From 516b922b38332874b2d9e57d2a1e22d18871b6c5 Mon Sep 17 00:00:00 2001 From: Erik Frojdh Date: Thu, 24 Sep 2026 09:09:37 +0200 Subject: [PATCH 1/5] updated licensing info --- AGENTS.md | 8 + CMakeLists.txt | 4 +- LICENSES/LGPL-2.1-or-later.txt | 459 ++++++++++++++++++ LICENSES/MIT.txt | 21 + README.md | 14 + RELEASE.md | 18 +- THIRD-PARTY-NOTICES.txt | 137 ++++++ cmake/FindSphinx.cmake | 2 + cmake/helpers.cmake | 1 + cmake/package_config.cmake | 2 + cmake/project-config.cmake.in | 1 + conda-recipe/meta.yaml | 7 +- docs/conf.py.in | 1 + etc/add_license.py | 21 +- etc/update_version.py | 1 - include/aare/ProducerConsumerQueue.hpp | 10 +- include/aare/ROI.hpp | 1 + include/aare/ROIGeometry.hpp | 1 + .../hist/PedestalTrackingPixelHistogram.hpp | 1 + include/aare/hist/PixelHistogram.hpp | 1 + include/aare/hist/PixelHistogramImpl.hpp | 1 + include/aare/utils/math_helpers.hpp | 1 + pyproject.toml | 5 +- python/aare/Cluster.py | 1 + python/aare/_version.py | 1 + python/src/bind_Defs.hpp | 1 + python/src/bind_Eta.hpp | 1 + python/src/module_config.hpp | 1 + python/tests/test_CtbRawFile.py | 1 + python/tests/test_FastPedestal.py | 1 + python/tests/test_InterpolationAPI.py | 1 + python/tests/test_Pedestal.py | 1 + python/tests/test_ROI.py | 1 + python/tests/test_RawMasterFile.py | 1 + python/tests/test_Transformation.py | 1 + python/tests/test_interpolation_simulated.py | 1 + src/Interpolation.test.cpp | 1 + src/NumpyHelpers.cpp | 2 +- src/ROI.cpp | 1 + src/ROI.test.cpp | 1 + src/ROIGeometry.cpp | 1 + src/hist/PedestalTrackingPixelHistogram.cpp | 1 + src/hist/PixelHistogram.test.cpp | 1 + src/hist/PixelHistogramImpl.test.cpp | 1 + src/to_string.cpp | 1 + tests/test_config.hpp.in | 1 + 46 files changed, 722 insertions(+), 20 deletions(-) create mode 100644 LICENSES/LGPL-2.1-or-later.txt create mode 100644 LICENSES/MIT.txt create mode 100644 THIRD-PARTY-NOTICES.txt diff --git a/AGENTS.md b/AGENTS.md index 7311e097..c4918858 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -188,6 +188,14 @@ headers never leak into the public API. Minuit2 is a private, namespace/module. - Start new source files with `SPDX-License-Identifier: MPL-2.0`, using the appropriate comment syntax. +- Code copied from other projects keeps its original license. Retain its + copyright and license notice, set the SPDX identifier to that license, and + list the license in `LICENSES/`, `pyproject.toml`, `conda-recipe/meta.yaml`, + and `README.md`. `etc/add_license.py` skips files with a license notice. +- When adding or bumping a fetched dependency that is compiled into the + Python extension, update its version and notice in + `THIRD-PARTY-NOTICES.txt` and the license expressions in `pyproject.toml` + and `conda-recipe/meta.yaml`. - Prefer descriptive names to comments that only restate the code. - Do not add a dependency unless the benefit justifies the packaging and deployment cost. diff --git a/CMakeLists.txt b/CMakeLists.txt index 89cb0e52..1998de15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -106,7 +106,7 @@ if(AARE_FETCH_MINUIT2) FetchContent_Declare( Minuit2 GIT_REPOSITORY https://github.com/GooFit/Minuit2.git - GIT_TAG master + GIT_TAG v6-40-02 UPDATE_DISCONNECTED 1) # Disable Minuit2 extras we don't need set(minuit2_mpi @@ -153,7 +153,7 @@ if(AARE_FETCH_ZMQ) FetchContent_Declare( libzmq GIT_REPOSITORY https://github.com/zeromq/libzmq.git - GIT_TAG v4.3.4 + GIT_TAG v4.3.5 PATCH_COMMAND ${ZMQ_PATCH_COMMAND} UPDATE_DISCONNECTED 1) # Disable unwanted options from libzmq diff --git a/LICENSES/LGPL-2.1-or-later.txt b/LICENSES/LGPL-2.1-or-later.txt new file mode 100644 index 00000000..7f01a06c --- /dev/null +++ b/LICENSES/LGPL-2.1-or-later.txt @@ -0,0 +1,459 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt new file mode 100644 index 00000000..7dfed9f2 --- /dev/null +++ b/LICENSES/MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright 2017-2023 Leon Merten Lohse + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index ca41ae10..7a9f1c8a 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,20 @@ Detailed documentation including installation can be found in [Documentation](ht This project is licensed under the MPL-2.0 license. See the LICENSE file or https://www.mozilla.org/en-US/MPL/ for details. +Two files contain code from other projects and keep their original licenses: + +- `include/aare/ProducerConsumerQueue.hpp` is Apache-2.0, from + [folly](https://github.com/facebook/folly) by Meta Platforms. +- `src/NumpyHelpers.cpp` is MIT, from + [libnpy](https://github.com/llohse/libnpy) by Leon Merten Lohse. + +The full license texts are in the `LICENSES` directory. + +The Python wheels and conda packages also contain Minuit2 +(LGPL-2.1-or-later), {fmt} and nlohmann/json (MIT), and pybind11 +(BSD-3-Clause), which CMake downloads at build time. See +`THIRD-PARTY-NOTICES.txt` for their notices. + ## Build and install Prerequisites diff --git a/RELEASE.md b/RELEASE.md index c5287bb4..0a897d00 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -87,10 +87,20 @@ ``std::invalid_argument`` in all build types. The previous assertion was compiled out of Release builds, so a zero-size queue overflowed its buffer on the first write. -- Corrected the license metadata for the vendored ``ProducerConsumerQueue``. - The header is tagged ``Apache-2.0`` like upstream folly, the MPL 2.0 and - Apache 2.0 texts ship in ``LICENSES/``, and the conda package declares - ``MPL-2.0 AND Apache-2.0`` with both license files. +- Corrected the license metadata for vendored third-party code. + ``ProducerConsumerQueue.hpp`` is tagged ``Apache-2.0`` like upstream folly + and ``NumpyHelpers.cpp`` is tagged ``MIT`` like upstream libnpy. The MPL 2.0, + Apache 2.0, and MIT texts ship in ``LICENSES/``, and both the conda package + and the Python wheel declare ``MPL-2.0 AND Apache-2.0 AND MIT`` with all + three license files. Building the wheel now requires scikit-build-core 0.11 + or newer. +- The wheel and conda package now ship ``THIRD-PARTY-NOTICES.txt`` and the + LGPL 2.1 text for the libraries compiled into the extension: Minuit2 + (LGPL-2.1-or-later), {fmt} and nlohmann/json (MIT), and pybind11 + (BSD-3-Clause). Their license expressions include these licenses. + Minuit2 is now fetched at tag ``v6-40-02`` instead of ``master``, and + libzmq at ``v4.3.5`` (MPL-2.0) instead of ``v4.3.4`` (LGPL-3.0 with a + static-linking exception). - ``RawFile`` and ``File`` reject raw files with frame padding disabled unless the frame discard policy is ``discardpartial``. The constructor reports the master path before opening data subfiles. Legacy ``.raw`` master files now diff --git a/THIRD-PARTY-NOTICES.txt b/THIRD-PARTY-NOTICES.txt new file mode 100644 index 00000000..3d07d9c7 --- /dev/null +++ b/THIRD-PARTY-NOTICES.txt @@ -0,0 +1,137 @@ +Third-party notices for aare binary packages +============================================ + +The aare Python wheels and conda packages compile the following libraries into +the aare extension module. They are not part of the aare source repository; +CMake downloads them at build time. Their license notices follow below. + + Minuit2 v6-40-02 LGPL-2.1-or-later https://github.com/GooFit/Minuit2 + {fmt} 12.1.0 MIT https://github.com/fmtlib/fmt + nlohmann/json 3.11.3 MIT https://github.com/nlohmann/json + pybind11 3.0.3 BSD-3-Clause https://github.com/pybind/pybind11 + +Minuit2 is linked statically. To use aare with a modified Minuit2, build aare +from source (https://github.com/slsdetectorgroup/aare) with +-DAARE_FETCH_MINUIT2=OFF and make your Minuit2 build findable by CMake's +find_package(Minuit2). + +------------------------------------------------------------------------------ +Minuit2 +------------------------------------------------------------------------------ + +Copyright (c) 2005 LCG ROOT Math team, CERN/PH-SFT + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +See LGPL2_1.txt next to this file for the text of the LGPL version 2.1. +For questions regarding the ROOT license, please contact rootdev@cern.ch. + +The precompiled binary distributions provided as a service to our community +may feature packages released with licenses different from LGPL. + +In aare packages the text of the LGPL version 2.1 is in +LICENSES/LGPL-2.1-or-later.txt. + +------------------------------------------------------------------------------ +{fmt} +------------------------------------------------------------------------------ + +Copyright (c) 2012 - present, Victor Zverovich and {fmt} contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Optional exception to the license --- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into a machine-executable object form of such +source code, you may redistribute such embedded portions in such object form +without including the above copyright and permission notices. + +------------------------------------------------------------------------------ +nlohmann/json +------------------------------------------------------------------------------ + +MIT License + +Copyright (c) 2013-2022 Niels Lohmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------ +pybind11 +------------------------------------------------------------------------------ + +Copyright (c) 2016 Wenzel Jakob , All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Please also refer to the file .github/CONTRIBUTING.md, which clarifies licensing of +external contributions to this project including patches, pull requests, etc. diff --git a/cmake/FindSphinx.cmake b/cmake/FindSphinx.cmake index a052e1d4..b980c93d 100644 --- a/cmake/FindSphinx.cmake +++ b/cmake/FindSphinx.cmake @@ -1,3 +1,5 @@ +# SPDX-License-Identifier: MPL-2.0 + # Look for an executable called sphinx-build find_program( SPHINX_EXECUTABLE diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 595f9530..2fe1d7cf 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 function(default_build_type val) if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) message(STATUS "No build type selected, default to Release") diff --git a/cmake/package_config.cmake b/cmake/package_config.cmake index d018a7f0..0d9c0dcb 100644 --- a/cmake/package_config.cmake +++ b/cmake/package_config.cmake @@ -1,3 +1,5 @@ +# SPDX-License-Identifier: MPL-2.0 + # This cmake code creates the configuration that is found and used by # find_package() of another cmake project diff --git a/cmake/project-config.cmake.in b/cmake/project-config.cmake.in index 793a373f..b3f0d569 100644 --- a/cmake/project-config.cmake.in +++ b/cmake/project-config.cmake.in @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 # Config file for @PROJECT_NAME_LOWER@ # # It defines the following variables: diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 70ec7076..d2d39b53 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -26,7 +26,7 @@ requirements: - python - pip - numpy >=2.1 - - scikit-build-core + - scikit-build-core >=0.11 - pybind11 >=3.0.3 - matplotlib # needed in host to solve the environment for run @@ -50,8 +50,11 @@ test: - python -m pytest python/tests about: - license: MPL-2.0 AND Apache-2.0 + license: MPL-2.0 AND Apache-2.0 AND MIT AND BSD-3-Clause AND LGPL-2.1-or-later license_file: - LICENSES/MPL-2.0.txt - LICENSES/Apache-2.0.txt + - LICENSES/MIT.txt + - LICENSES/LGPL-2.1-or-later.txt + - THIRD-PARTY-NOTICES.txt summary: Data analysis library for hybrid pixel detectors from PSI diff --git a/docs/conf.py.in b/docs/conf.py.in index ad735752..1e0b5a71 100644 --- a/docs/conf.py.in +++ b/docs/conf.py.in @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full diff --git a/etc/add_license.py b/etc/add_license.py index f43b6cdb..ca02ba43 100644 --- a/etc/add_license.py +++ b/etc/add_license.py @@ -4,13 +4,17 @@ import fnmatch import os from pathlib import Path -CPP_PATTERNS = ["*.h", "*.hpp", "*.cpp"] -PY_PATTERNS = ["*.py"] -CMAKE_PATTERNS = ["CMakeLists.txt"] +CPP_PATTERNS = ["*.h", "*.hpp", "*.cpp", "*.hpp.in"] +PY_PATTERNS = ["*.py", "*.py.in"] +CMAKE_PATTERNS = ["CMakeLists.txt", "*.cmake", "*.cmake.in"] FILE_PATTERNS = CPP_PATTERNS + PY_PATTERNS + CMAKE_PATTERNS LICENSE_TEXT = "SPDX-License-Identifier: MPL-2.0" +# Text found in license notices of code copied from other projects. Such files +# keep their original license and must be tagged by hand. +THIRD_PARTY_MARKERS = ["Copyright", "Permission is hereby granted", "Licensed under"] + def get_comment_prefix(filename: str) -> str | None: if any(fnmatch.fnmatch(filename, p) for p in CPP_PATTERNS): @@ -43,6 +47,10 @@ def process_file(filepath: Path) -> bool: if any("SPDX-License-Identifier" in line for line in lines): return False + if any(marker in line for line in lines for marker in THIRD_PARTY_MARKERS): + print(f"⚠️ Skipped {filepath}: contains a license notice, tag it by hand") + return False + insert_index = 0 # For Python, keep shebang on the very first line @@ -50,6 +58,11 @@ def process_file(filepath: Path) -> bool: if lines[0].startswith("#!"): insert_index = 1 + # cmake-format merges consecutive comment lines into one paragraph + if any(fnmatch.fnmatch(filename, p) for p in CMAKE_PATTERNS): + if len(lines) > insert_index and lines[insert_index].startswith("#"): + lines.insert(insert_index, "\n") + lines.insert(insert_index, license_line) try: @@ -69,7 +82,7 @@ def main() -> None: parser.add_argument( "path", help="Root directory to recursively process " - "(*.h, *.cpp, *.py, and CMakeLists.txt).", + "(C++, Python, and CMake sources, and their .in templates).", ) args = parser.parse_args() diff --git a/etc/update_version.py b/etc/update_version.py index 78eea2a3..a939d9f7 100644 --- a/etc/update_version.py +++ b/etc/update_version.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: MPL-2.0 -# Copyright (C) 2021 Contributors to the Aare Package """ Script to update VERSION file with semantic versioning if provided as an argument, or with 0.0.0 if no argument is provided. """ diff --git a/include/aare/ProducerConsumerQueue.hpp b/include/aare/ProducerConsumerQueue.hpp index dc5670a5..d630490f 100644 --- a/include/aare/ProducerConsumerQueue.hpp +++ b/include/aare/ProducerConsumerQueue.hpp @@ -18,10 +18,12 @@ // @author Bo Hu (bhu@fb.com) // @author Jordan DeLong (delong.j@fb.com) -// Changes made by the PSD Detector Group (PSI) relative to -// folly/ProducerConsumerQueue.h: -// - Copied constexpr hardware_destructive_interference_size = 128 from -// folly/lang/Align.h into this file. +// Changes made by the CPS Detector Group at PSI relative to +// folly/ProducerConsumerQueue.h in folly v2024.03.18.00 (commit 8aceb9bf727c; +// the file is identical in v2022.01.03.00 through v2024.03.18.00): +// - Replaced the include of folly/concurrency/CacheLocality.h with a local +// constexpr hardware_destructive_interference_size = 128, the value that +// folly/lang/Align.h uses except on ARM and s390x. // - Changed the file extension to .hpp and the namespace to aare. // - Added a default constructor equivalent to ProducerConsumerQueue(2). // - Declared the move constructor and move assignment deleted. The queue is diff --git a/include/aare/ROI.hpp b/include/aare/ROI.hpp index 81c2a84e..36838f43 100644 --- a/include/aare/ROI.hpp +++ b/include/aare/ROI.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include "aare/defs.hpp" #include diff --git a/include/aare/ROIGeometry.hpp b/include/aare/ROIGeometry.hpp index 739b24c3..a8a90c37 100644 --- a/include/aare/ROIGeometry.hpp +++ b/include/aare/ROIGeometry.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include "aare/DetectorGeometry.hpp" #include "aare/ROI.hpp" diff --git a/include/aare/hist/PedestalTrackingPixelHistogram.hpp b/include/aare/hist/PedestalTrackingPixelHistogram.hpp index 88ca7f2f..2025d053 100644 --- a/include/aare/hist/PedestalTrackingPixelHistogram.hpp +++ b/include/aare/hist/PedestalTrackingPixelHistogram.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include "aare/FastPedestal.hpp" #include "aare/NDArray.hpp" diff --git a/include/aare/hist/PixelHistogram.hpp b/include/aare/hist/PixelHistogram.hpp index 05432d65..c9b199b9 100644 --- a/include/aare/hist/PixelHistogram.hpp +++ b/include/aare/hist/PixelHistogram.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include "aare/NDArray.hpp" #include "aare/NDView.hpp" diff --git a/include/aare/hist/PixelHistogramImpl.hpp b/include/aare/hist/PixelHistogramImpl.hpp index d68795c8..9af42fe1 100644 --- a/include/aare/hist/PixelHistogramImpl.hpp +++ b/include/aare/hist/PixelHistogramImpl.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once /* Basic pixel histogram class with templated axis and storage type. diff --git a/include/aare/utils/math_helpers.hpp b/include/aare/utils/math_helpers.hpp index 9481bd8a..6d9b3c80 100644 --- a/include/aare/utils/math_helpers.hpp +++ b/include/aare/utils/math_helpers.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include diff --git a/pyproject.toml b/pyproject.toml index 8fa52bde..6054356c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ regex = '^(?P\d+(?:\.\d+)*(?:[\.\+\w]+)?)$' result = "{version}" [build-system] -requires = ["scikit-build-core>=0.10", "pybind11", "numpy"] +requires = ["scikit-build-core>=0.11", "pybind11", "numpy"] build-backend = "scikit_build_core.build" [project] @@ -16,7 +16,8 @@ dependencies = [ "numpy", "matplotlib", ] -license = { file = "LICENSE" } +license = "MPL-2.0 AND Apache-2.0 AND MIT AND BSD-3-Clause AND LGPL-2.1-or-later" +license-files = ["LICENSES/*", "THIRD-PARTY-NOTICES.txt"] diff --git a/python/aare/Cluster.py b/python/aare/Cluster.py index 7946567d..2da800b9 100644 --- a/python/aare/Cluster.py +++ b/python/aare/Cluster.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 from . import _aare import numpy as np from .factory import _type_to_char diff --git a/python/aare/_version.py b/python/aare/_version.py index f3530c4c..c2bab73f 100644 --- a/python/aare/_version.py +++ b/python/aare/_version.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 # This file is used to get the version of the package from the VERSION file from importlib.metadata import PackageNotFoundError, version from pathlib import Path diff --git a/python/src/bind_Defs.hpp b/python/src/bind_Defs.hpp index c6e7d185..c6592f24 100644 --- a/python/src/bind_Defs.hpp +++ b/python/src/bind_Defs.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include #include diff --git a/python/src/bind_Eta.hpp b/python/src/bind_Eta.hpp index 716b0e72..e2b9302e 100644 --- a/python/src/bind_Eta.hpp +++ b/python/src/bind_Eta.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/CalculateEta.hpp" #include "aare/to_string.hpp" diff --git a/python/src/module_config.hpp b/python/src/module_config.hpp index e2f4e8a7..ad5d7ca1 100644 --- a/python/src/module_config.hpp +++ b/python/src/module_config.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include diff --git a/python/tests/test_CtbRawFile.py b/python/tests/test_CtbRawFile.py index 6485b076..e23c99f2 100644 --- a/python/tests/test_CtbRawFile.py +++ b/python/tests/test_CtbRawFile.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import pytest diff --git a/python/tests/test_FastPedestal.py b/python/tests/test_FastPedestal.py index f93560cb..1feacaea 100644 --- a/python/tests/test_FastPedestal.py +++ b/python/tests/test_FastPedestal.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import numpy as np import pytest diff --git a/python/tests/test_InterpolationAPI.py b/python/tests/test_InterpolationAPI.py index e110087a..48686253 100644 --- a/python/tests/test_InterpolationAPI.py +++ b/python/tests/test_InterpolationAPI.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import pytest from aare import Interpolator, ClusterVector, Etai, Cluster diff --git a/python/tests/test_Pedestal.py b/python/tests/test_Pedestal.py index 27f9897a..4248d157 100644 --- a/python/tests/test_Pedestal.py +++ b/python/tests/test_Pedestal.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import numpy as np import pytest diff --git a/python/tests/test_ROI.py b/python/tests/test_ROI.py index f0b98de0..3a117016 100644 --- a/python/tests/test_ROI.py +++ b/python/tests/test_ROI.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import pytest import numpy as np diff --git a/python/tests/test_RawMasterFile.py b/python/tests/test_RawMasterFile.py index 5ad68b48..94669d77 100644 --- a/python/tests/test_RawMasterFile.py +++ b/python/tests/test_RawMasterFile.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import json diff --git a/python/tests/test_Transformation.py b/python/tests/test_Transformation.py index 2b381085..0d91c10a 100644 --- a/python/tests/test_Transformation.py +++ b/python/tests/test_Transformation.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import pytest import numpy as np from aare import CtbRawFile, transform diff --git a/python/tests/test_interpolation_simulated.py b/python/tests/test_interpolation_simulated.py index 5a3cdf3c..57a6ab63 100644 --- a/python/tests/test_interpolation_simulated.py +++ b/python/tests/test_interpolation_simulated.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 #test script to test interpolation on simulated data diff --git a/src/Interpolation.test.cpp b/src/Interpolation.test.cpp index 41a8b70a..764de414 100644 --- a/src/Interpolation.test.cpp +++ b/src/Interpolation.test.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/ClusterVector.hpp" #include "aare/Interpolator.hpp" #include "aare/NDArray.hpp" diff --git a/src/NumpyHelpers.cpp b/src/NumpyHelpers.cpp index a5448338..75462953 100644 --- a/src/NumpyHelpers.cpp +++ b/src/NumpyHelpers.cpp @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: MIT /* 28-03-2024 modified by: Bechir Braham diff --git a/src/ROI.cpp b/src/ROI.cpp index 9cb37961..7e91c49d 100644 --- a/src/ROI.cpp +++ b/src/ROI.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/ROI.hpp" #include "aare/DetectorGeometry.hpp" #include "aare/ROIGeometry.hpp" diff --git a/src/ROI.test.cpp b/src/ROI.test.cpp index 6cc7bb26..9bbe4dff 100644 --- a/src/ROI.test.cpp +++ b/src/ROI.test.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/ROI.hpp" #include diff --git a/src/ROIGeometry.cpp b/src/ROIGeometry.cpp index f61f9079..8008118d 100644 --- a/src/ROIGeometry.cpp +++ b/src/ROIGeometry.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/ROIGeometry.hpp" #include diff --git a/src/hist/PedestalTrackingPixelHistogram.cpp b/src/hist/PedestalTrackingPixelHistogram.cpp index f02004ab..b97b5ad9 100644 --- a/src/hist/PedestalTrackingPixelHistogram.cpp +++ b/src/hist/PedestalTrackingPixelHistogram.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/hist/PedestalTrackingPixelHistogram.hpp" #include "aare/File.hpp" #include "aare/MultiThreadedFileReader.hpp" diff --git a/src/hist/PixelHistogram.test.cpp b/src/hist/PixelHistogram.test.cpp index d681c789..50b1fcec 100644 --- a/src/hist/PixelHistogram.test.cpp +++ b/src/hist/PixelHistogram.test.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include #include diff --git a/src/hist/PixelHistogramImpl.test.cpp b/src/hist/PixelHistogramImpl.test.cpp index f3526c17..f9456efa 100644 --- a/src/hist/PixelHistogramImpl.test.cpp +++ b/src/hist/PixelHistogramImpl.test.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include #include diff --git a/src/to_string.cpp b/src/to_string.cpp index 8776c501..2753695f 100644 --- a/src/to_string.cpp +++ b/src/to_string.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/to_string.hpp" namespace aare { diff --git a/tests/test_config.hpp.in b/tests/test_config.hpp.in index e314b8fe..8030213e 100644 --- a/tests/test_config.hpp.in +++ b/tests/test_config.hpp.in @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include #include From 691735d89b071dafd721749c6914fe6c9a839c4b Mon Sep 17 00:00:00 2001 From: Erik Frojdh Date: Thu, 24 Sep 2026 15:23:07 +0200 Subject: [PATCH 2/5] added missing alias for Minuit standalone --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1998de15..b0bc911d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,6 +129,7 @@ if(AARE_FETCH_MINUIT2) add_library(aare::Minuit2 ALIAS Minuit2) else() find_package(Minuit2 CONFIG QUIET) + add_library(aare::Minuit2 ALIAS Minuit2::Minuit2) if(NOT Minuit2_FOUND) message( STATUS From 24f1a6138a8b267243bbfc7c986acbd756c6c2c6 Mon Sep 17 00:00:00 2001 From: Erik Frojdh Date: Thu, 24 Sep 2026 15:24:30 +0200 Subject: [PATCH 3/5] now correct --- CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b0bc911d..e75ea5d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,8 +129,9 @@ if(AARE_FETCH_MINUIT2) add_library(aare::Minuit2 ALIAS Minuit2) else() find_package(Minuit2 CONFIG QUIET) - add_library(aare::Minuit2 ALIAS Minuit2::Minuit2) - if(NOT Minuit2_FOUND) + if(Minuit2_FOUND) + add_library(aare::Minuit2 ALIAS Minuit2::Minuit2) + else() message( STATUS "Stand alone Minuit2 not found. Trying to find it as a part of ROOT") From 8579a140f718db149841d482129a612cffa86f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Fr=C3=B6jdh?= Date: Fri, 25 Sep 2026 10:43:58 +0200 Subject: [PATCH 4/5] Improved ArrayExpr (#378) - removes code repetition - avoid temporaries for scalar operations --- RELEASE.md | 7 ++ benchmarks/ndarray_benchmark.cpp | 76 +++++++++++++++ include/aare/ArrayExpr.hpp | 153 +++++++++++++++++-------------- include/aare/NDArray.hpp | 151 +++++++++++++----------------- include/aare/NDView.hpp | 15 +-- src/NDArray.test.cpp | 55 ++++++++++- 6 files changed, 290 insertions(+), 167 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 0a897d00..256cbb63 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -29,9 +29,14 @@ - Added string representator in python for Cluster and Eta - Added roi slice method in python for easy slicing of numpy arrays ``array[roi.slice()]``. - added context manager for ``aare.RawMasterFile`` +- ``NDArray``/``NDView`` expressions now support scalar operands + (``2 * a + b / 4``) and can be assigned to an existing ``NDArray``, reusing + its buffer. ### API Changes: +- ``NDArray`` ``+ - * /`` with a scalar now returns a lazy expression instead + of an ``NDArray`` and no longer converts the scalar to the element type. - ``FastPedestal`` variance is now a private ``double`` intermediate. Removed the C++ ``variance()``/``variance_unchecked()`` APIs and Python ``var()``. Standard deviation is calculated before conversion to the output type, @@ -77,6 +82,8 @@ ### Bugfixes: - Mismatched operators inhibited vectorization in gcc of NDArray math operators +- ``NDArray``/``NDView`` math operators and expressions were not vectorized + for ``uint8_t`` and 64 bit integers, up to 30x slower than a plain loop. - Removed the move constructor and move assignment of ``ProducerConsumerQueue``. They left the moved-from queue with a null buffer, crashing its destructor if it still held elements, and leaked diff --git a/benchmarks/ndarray_benchmark.cpp b/benchmarks/ndarray_benchmark.cpp index 456f4d40..b3a3f610 100644 --- a/benchmarks/ndarray_benchmark.cpp +++ b/benchmarks/ndarray_benchmark.cpp @@ -130,4 +130,80 @@ BENCHMARK_F(TwoArrays, MultiplyAddDivideWithIndex)(benchmark::State &st) { } } +BENCHMARK_F(TwoArrays, ScalarMultiplyAddWithOperator)(benchmark::State &st) { + for (auto _ : st) { + // This code gets timed + NDArray res = a * 2 + b; + benchmark::DoNotOptimize(res); + } +} +BENCHMARK_F(TwoArrays, ScalarMultiplyAddWithIndex)(benchmark::State &st) { + for (auto _ : st) { + // This code gets timed + NDArray res(a.shape()); + for (uint32_t i = 0; i < a.size(); i++) { + res(i) = a(i) * 2 + b(i); + } + benchmark::DoNotOptimize(res); + } +} + +BENCHMARK_F(TwoArrays, AddToExistingWithOperator)(benchmark::State &st) { + NDArray res(a.shape()); + for (auto _ : st) { + // This code gets timed + res = a + b; + benchmark::DoNotOptimize(res); + } +} +BENCHMARK_F(TwoArrays, AddToExistingWithIndex)(benchmark::State &st) { + NDArray res(a.shape()); + for (auto _ : st) { + // This code gets timed + for (uint32_t i = 0; i < a.size(); i++) { + res(i) = a(i) + b(i); + } + benchmark::DoNotOptimize(res); + } +} + +// A store to uint8_t or a 64 bit integer can alias the members of NDArray. +// Compare with a loop over raw pointers to check that this does not block +// vectorization. +template void InPlaceAddWithOperator(benchmark::State &st) { + NDArray a({size, size}, 1); + NDArray b({size, size}, 2); + for (auto _ : st) { + a += b; + benchmark::DoNotOptimize(a); + } +} +template void InPlaceAddWithPointer(benchmark::State &st) { + NDArray a({size, size}, 1); + NDArray b({size, size}, 2); + for (auto _ : st) { + T *dst = a.data(); + const T *src = b.data(); + for (ssize_t i = 0, n = a.size(); i < n; ++i) { + dst[i] += src[i]; + } + benchmark::DoNotOptimize(a); + } +} +template void AddToExistingWithOperator(benchmark::State &st) { + NDArray a({size, size}, 1); + NDArray b({size, size}, 2); + NDArray res({size, size}, 0); + for (auto _ : st) { + res = a + b; + benchmark::DoNotOptimize(res); + } +} +BENCHMARK_TEMPLATE(InPlaceAddWithOperator, uint8_t); +BENCHMARK_TEMPLATE(InPlaceAddWithPointer, uint8_t); +BENCHMARK_TEMPLATE(AddToExistingWithOperator, uint8_t); +BENCHMARK_TEMPLATE(InPlaceAddWithOperator, int64_t); +BENCHMARK_TEMPLATE(InPlaceAddWithPointer, int64_t); +BENCHMARK_TEMPLATE(AddToExistingWithOperator, int64_t); + BENCHMARK_MAIN(); \ No newline at end of file diff --git a/include/aare/ArrayExpr.hpp b/include/aare/ArrayExpr.hpp index 8eab0d2e..8520208d 100644 --- a/include/aare/ArrayExpr.hpp +++ b/include/aare/ArrayExpr.hpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include namespace aare { @@ -20,80 +22,95 @@ template class ArrayExpr { } }; -template -class ArrayAdd : public ArrayExpr, Ndim> { - const A &arr1_; - const B &arr2_; - - public: - ArrayAdd(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) { - assert(arr1.size() == arr2.size()); - } - auto operator[](size_t i) const { return arr1_[i] + arr2_[i]; } - size_t size() const { return arr1_.size(); } - std::array shape() const { return arr1_.shape(); } -}; - -template -class ArraySub : public ArrayExpr, Ndim> { - const A &arr1_; - const B &arr2_; - - public: - ArraySub(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) { - assert(arr1.size() == arr2.size()); - } - auto operator[](size_t i) const { return arr1_[i] - arr2_[i]; } - size_t size() const { return arr1_.size(); } - std::array shape() const { return arr1_.shape(); } -}; - -template -class ArrayMul : public ArrayExpr, Ndim> { - const A &arr1_; - const B &arr2_; - - public: - ArrayMul(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) { - assert(arr1.size() == arr2.size()); - } - auto operator[](size_t i) const { return arr1_[i] * arr2_[i]; } - size_t size() const { return arr1_.size(); } - std::array shape() const { return arr1_.shape(); } -}; - -template -class ArrayDiv : public ArrayExpr, Ndim> { - const A &arr1_; - const B &arr2_; - - public: - ArrayDiv(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) { - assert(arr1.size() == arr2.size()); - } - auto operator[](size_t i) const { return arr1_[i] / arr2_[i]; } - size_t size() const { return arr1_.size(); } - std::array shape() const { return arr1_.shape(); } -}; - -template -auto operator+(const ArrayExpr &arr1, const ArrayExpr &arr2) { - return ArrayAdd, ArrayExpr, Ndim>(arr1, arr2); +/** + * @brief What an expression stores for one of its operands. An NDArray is + * replaced by a view, everything else is copied. The expression then holds its + * data pointers by value, so the compiler can keep them in registers even if + * the evaluation writes to a type that may alias them (e.g. uint8_t). + */ +template +auto operand(const ArrayExpr &expr) { + if constexpr (E::is_leaf) + return static_cast(expr).view(); + else + return static_cast(expr); } -template -auto operator-(const ArrayExpr &arr1, const ArrayExpr &arr2) { - return ArraySub, ArrayExpr, Ndim>(arr1, arr2); +template +class ArrayBinaryOp : public ArrayExpr, Ndim> { + A arr1_; + B arr2_; + + public: + ArrayBinaryOp(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) { + assert(arr1.shape() == arr2.shape()); + } + auto operator[](size_t i) const { return Op{}(arr1_[i], arr2_[i]); } + size_t size() const { return arr1_.size(); } + std::array shape() const { return arr1_.shape(); } +}; + +/** @brief Scalar operand, takes size and shape from the expression it is + * combined with. */ +template +class ArrayScalar : public ArrayExpr, Ndim> { + T value_; + size_t size_; + std::array shape_; + + public: + template + ArrayScalar(T value, const ArrayExpr &like) + : value_(value), size_(like.size()), shape_(like.shape()) {} + T operator[](size_t) const { return value_; } + size_t size() const { return size_; } + std::array shape() const { return shape_; } +}; + +// Builds the expression for expr op expr, expr op scalar and scalar op expr +template +auto binary_op(const ArrayExpr &lhs, const ArrayExpr &rhs) { + auto a = operand(lhs); + auto b = operand(rhs); + return ArrayBinaryOp(a, b); } -template -auto operator*(const ArrayExpr &arr1, const ArrayExpr &arr2) { - return ArrayMul, ArrayExpr, Ndim>(arr1, arr2); +template >> +auto binary_op(const ArrayExpr &lhs, T rhs) { + return binary_op(lhs, ArrayScalar(rhs, lhs)); } -template -auto operator/(const ArrayExpr &arr1, const ArrayExpr &arr2) { - return ArrayDiv, ArrayExpr, Ndim>(arr1, arr2); +template >> +auto binary_op(T lhs, const ArrayExpr &rhs) { + return binary_op(ArrayScalar(lhs, rhs), rhs); +} + +// The operators only take part in overload resolution if binary_op accepts +// the operands, meaning at least one of them is an ArrayExpr +template +auto operator+(const L &lhs, const R &rhs) + -> decltype(binary_op>(lhs, rhs)) { + return binary_op>(lhs, rhs); +} + +template +auto operator-(const L &lhs, const R &rhs) + -> decltype(binary_op>(lhs, rhs)) { + return binary_op>(lhs, rhs); +} + +template +auto operator*(const L &lhs, const R &rhs) + -> decltype(binary_op>(lhs, rhs)) { + return binary_op>(lhs, rhs); +} + +template +auto operator/(const L &lhs, const R &rhs) + -> decltype(binary_op>(lhs, rhs)) { + return binary_op>(lhs, rhs); } } // namespace aare \ No newline at end of file diff --git a/include/aare/NDArray.hpp b/include/aare/NDArray.hpp index fb6c3bb5..68f88ad6 100644 --- a/include/aare/NDArray.hpp +++ b/include/aare/NDArray.hpp @@ -27,6 +27,8 @@ class NDArray : public ArrayExpr, Ndim> { T *data_; public: + static constexpr bool is_leaf = true; + /////////////////////////////////////////////////////////////////////////////// // Constructors // @@ -135,9 +137,7 @@ class NDArray : public ArrayExpr, Ndim> { */ template NDArray(ArrayExpr &&expr) : NDArray(expr.shape()) { - for (size_t i = 0; i < size_; ++i) { - data_[i] = expr[i]; - } + *this = expr; } /** @@ -324,6 +324,26 @@ class NDArray : public ArrayExpr, Ndim> { return *this; } + /** + * @brief Evaluate an ArrayExpr into the NDArray. Reuses the existing + * buffer if the shape matches. + */ + template NDArray &operator=(const ArrayExpr &expr) { + if (expr.shape() != shape_) { + // Evaluate before releasing the buffer, expr might refer to it + NDArray tmp(expr.shape()); + tmp = expr; + return *this = std::move(tmp); + } + // Local copies, see elementwise() + const auto e = operand(expr); + T *data = data_; + for (size_t i = 0, n = size_; i < n; ++i) { + data[i] = e[i]; + } + return *this; + } + /////////////////////////////////////////////////////////////////////////////// // Math operators // @@ -337,10 +357,7 @@ class NDArray : public ArrayExpr, Ndim> { throw(std::runtime_error( "Shape of NDArray must match for operator +=")); - for (size_t i = 0; i < size_; ++i) { - data_[i] += other.data_[i]; - } - return *this; + return elementwise(other.data_, std::plus()); } /** @@ -351,10 +368,7 @@ class NDArray : public ArrayExpr, Ndim> { throw(std::runtime_error( "Shape of NDArray must match for operator -=")); - for (size_t i = 0; i < size_; ++i) { - data_[i] -= other.data_[i]; - } - return *this; + return elementwise(other.data_, std::minus()); } /** @@ -365,10 +379,7 @@ class NDArray : public ArrayExpr, Ndim> { throw(std::runtime_error( "Shape of NDArray must match for operator *=")); - for (size_t i = 0; i < size_; ++i) { - data_[i] *= other.data_[i]; - } - return *this; + return elementwise(other.data_, std::multiplies()); } /** @@ -379,13 +390,9 @@ class NDArray : public ArrayExpr, Ndim> { */ template NDArray &operator/=(const NDArray &other) { // check shape - if (shape_ == other.shape()) { - for (size_t i = 0; i < size_; ++i) { - data_[i] /= other(i); - } - return *this; - } - throw(std::runtime_error("Shape of NDArray must match")); + if (shape_ != other.shape()) + throw(std::runtime_error("Shape of NDArray must match")); + return elementwise(other.data(), std::divides<>()); } /** @@ -400,36 +407,28 @@ class NDArray : public ArrayExpr, Ndim> { * @brief Add a scalar value to all elements in the NDArray. */ NDArray &operator+=(const T &value) { - for (size_t i = 0; i < size_; ++i) - data_[i] += value; - return *this; + return elementwise(value, std::plus()); } /** * @brief Subtract a scalar value to all elements in the NDArray. */ NDArray &operator-=(const T &value) { - for (size_t i = 0; i < size_; ++i) - data_[i] -= value; - return *this; + return elementwise(value, std::minus()); } /** * @brief Multiply all elements in the NDArray with a scalar value */ NDArray &operator*=(const T &value) { - for (size_t i = 0; i < size_; ++i) - data_[i] *= value; - return *this; + return elementwise(value, std::multiplies()); } /** * @brief Divide all elements in the NDArray with a scalar value */ NDArray &operator/=(const T &value) { - for (size_t i = 0; i < size_; ++i) - data_[i] /= value; - return *this; + return elementwise(value, std::divides()); } /** @@ -437,53 +436,7 @@ class NDArray : public ArrayExpr, Ndim> { * Used for example to mask out gain bits for Jungfrau detectors. */ NDArray &operator&=(const T &mask) { - for (auto it = begin(); it != end(); ++it) - *it &= mask; - return *this; - } - - /** - * @brief Operator + with a scalar value. Returns a new NDArray. - * - * TODO! Expression template version of this? - */ - NDArray operator+(const T &value) { - NDArray result = *this; - result += value; - return result; - } - - /** - * @brief Operator - with a scalar value. Returns a new NDArray. - * - * TODO! Expression template version of this? - */ - NDArray operator-(const T &value) { - NDArray result = *this; - result -= value; - return result; - } - - /** - * @brief Operator * with a scalar value. Returns a new NDArray. - * - * TODO! Expression template version of this? - */ - NDArray operator*(const T &value) { - NDArray result = *this; - result *= value; - return result; - } - - /** - * @brief Operator / with a scalar value. Returns a new NDArray. - * - * TODO! Expression template version of this? - */ - NDArray operator/(const T &value) { - NDArray result = *this; - result /= value; - return result; + return elementwise(mask, std::bit_and()); } /** @@ -509,19 +462,16 @@ class NDArray : public ArrayExpr, Ndim> { * @brief Compute the square root of all elements in the NDArray. */ void sqrt() { - for (size_t i = 0; i < size_; ++i) { - data_[i] = std::sqrt(data_[i]); + T *data = data_; + for (size_t i = 0, n = size_; i < n; ++i) { + data[i] = std::sqrt(data[i]); } } /* * @brief Prefix increment operator. Increments all elements by 1. */ - NDArray &operator++() { - for (size_t i = 0; i < size_; ++i) - data_[i] += T{1}; - return *this; - } + NDArray &operator++() { return elementwise(T{1}, std::plus()); } /** @brief Create a mutable view of the NDArray. */ NDView view() { return NDView{data_, shape_}; } @@ -532,6 +482,31 @@ class NDArray : public ArrayExpr, Ndim> { } private: + /** + * @brief Apply data_[i] = op(data_[i], value) to all elements. + * + * The members are copied to locals since a store to data_[i] may alias + * them for some T (uint8_t, int64_t, ...). The compiler then has to reload + * them every iteration, which blocks vectorization. + */ + template NDArray &elementwise(T value, Op op) { + T *data = data_; + for (size_t i = 0, n = size_; i < n; ++i) { + data[i] = op(data[i], value); + } + return *this; + } + + /** @brief Apply data_[i] = op(data_[i], other[i]) to all elements. */ + template + NDArray &elementwise(const V *other, Op op) { + T *data = data_; + for (size_t i = 0, n = size_; i < n; ++i) { + data[i] = op(data[i], other[i]); + } + return *this; + } + /** * @brief Reset the NDArray to an empty state. Dropping the ownership of * the data. Used internally for move operations to avoid double free or diff --git a/include/aare/NDView.hpp b/include/aare/NDView.hpp index c507b41f..715127c9 100644 --- a/include/aare/NDView.hpp +++ b/include/aare/NDView.hpp @@ -173,8 +173,7 @@ class NDView : public ArrayExpr, Ndim> { } NDView &operator=(const T val) { - for (auto it = begin(); it != end(); ++it) - *it = val; + std::fill(begin(), end(), val); return *this; } @@ -232,15 +231,19 @@ class NDView : public ArrayExpr, Ndim> { template NDView &elemenwise(T val, BinaryOperation op) { - for (uint64_t i = 0; i != size_; ++i) { - buffer_[i] = op(buffer_[i], val); + // Local copies since a store to buffer_[i] may alias the members + T *buffer = buffer_; + for (uint64_t i = 0, n = size_; i != n; ++i) { + buffer[i] = op(buffer[i], val); } return *this; } template NDView &elemenwise(const NDView &other, BinaryOperation op) { - for (uint64_t i = 0; i != size_; ++i) { - buffer_[i] = op(buffer_[i], other.buffer_[i]); + T *buffer = buffer_; + const T *other_buffer = other.buffer_; + for (uint64_t i = 0, n = size_; i != n; ++i) { + buffer[i] = op(buffer[i], other_buffer[i]); } return *this; } diff --git a/src/NDArray.test.cpp b/src/NDArray.test.cpp index 3a3cafc0..3657c7b0 100644 --- a/src/NDArray.test.cpp +++ b/src/NDArray.test.cpp @@ -412,7 +412,7 @@ TEST_CASE("Elementwise operations on images") { NDArray A(shape, a_val); NDArray B(shape, b_val); double v = 1.0; - auto C = A - v; + NDArray C = A - v; REQUIRE(C.data() != A.data()); // Value of C matches @@ -429,7 +429,7 @@ TEST_CASE("Elementwise operations on images") { NDArray A(shape, a_val); NDArray B(shape, b_val); double v = 1.0; - auto C = A + v; + NDArray C = A + v; REQUIRE(C.data() != A.data()); // Value of C matches @@ -446,7 +446,7 @@ TEST_CASE("Elementwise operations on images") { NDArray A(shape, a_val); NDArray B(shape, b_val); double v = 3.7; - auto C = A / v; + NDArray C = A / v; REQUIRE(C.data() != A.data()); // Value of C matches @@ -463,12 +463,12 @@ TEST_CASE("Elementwise operations on images") { NDArray A(shape, a_val); NDArray B(shape, b_val); double v = 3.7; - auto C = A / v; + NDArray C = A * v; REQUIRE(C.data() != A.data()); // Value of C matches for (uint32_t i = 0; i < C.size(); ++i) { - REQUIRE(C(i) == a_val / v); + REQUIRE(C(i) == a_val * v); } // Value of A is not changed @@ -478,6 +478,51 @@ TEST_CASE("Elementwise operations on images") { } } +TEST_CASE("Expressions with scalars") { + NDArray A({5, 5}, 3); + NDArray B({5, 5}, 8); + + NDArray C = 2 * A + B / 4 - 1; + for (ssize_t i = 0; i < C.size(); ++i) { + REQUIRE(C(i) == 7); + } + + // The scalar is not converted to the element type of the array + NDArray D = A * 0.5; + for (ssize_t i = 0; i < D.size(); ++i) { + REQUIRE(D(i) == 1.5); + } +} + +TEST_CASE("Assign an expression to an existing NDArray") { + NDArray A({5, 5}, 3); + NDArray B({5, 5}, 8); + + SECTION("same shape reuses the buffer") { + NDArray C({5, 5}, 0); + auto *ptr = C.data(); + C = A + B; + REQUIRE(C.data() == ptr); + for (ssize_t i = 0; i < C.size(); ++i) { + REQUIRE(C(i) == 11); + } + } + SECTION("the target can be part of the expression") { + A = A + B; + for (ssize_t i = 0; i < A.size(); ++i) { + REQUIRE(A(i) == 11); + } + } + SECTION("different shape reallocates") { + NDArray C({2, 3}, 0); + C = A + B; + REQUIRE(C.shape() == A.shape()); + for (ssize_t i = 0; i < C.size(); ++i) { + REQUIRE(C(i) == 11); + } + } +} + TEST_CASE("Assign an std::array to a 1D NDArray") { NDArray a{{5}, 0}; std::array b{1, 2, 3, 4, 5}; From 941325ea151e627f61c9e732e34a751ecfffa510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Fr=C3=B6jdh?= Date: Fri, 25 Sep 2026 14:40:33 +0200 Subject: [PATCH 5/5] More uniform binding of Numpy arrays (#381) - Introduce a standard way to bind numpy arrays: py::array_t and .noconvert() on the argument - Take py::array_t by value since it's just a small wrapper - Update bindings --- RELEASE.md | 26 ++ docs/src/NumpyArrays.rst | 147 ++++++++++ docs/src/index.rst | 1 + docs/src/pyFit.rst | 10 + include/aare/NDArray.hpp | 15 +- include/aare/decode.hpp | 17 ++ python/src/bind_Cluster.hpp | 11 +- python/src/bind_ClusterFile.hpp | 10 +- python/src/bind_ClusterFinder.hpp | 16 +- python/src/bind_ClusterFinderMT.hpp | 16 +- python/src/bind_ClusterVector.hpp | 3 +- python/src/bind_FastPedestal.hpp | 6 +- python/src/bind_Interpolator.hpp | 19 +- python/src/bind_Pedestal.hpp | 17 +- .../bind_PedestalTrackingPixelHistogram.hpp | 6 +- python/src/bind_PixelHistogram.hpp | 2 +- python/src/bind_calibration.hpp | 32 +-- python/src/ctb_raw_file.hpp | 254 +++++++----------- python/src/fit.hpp | 18 +- python/src/var_cluster.hpp | 58 ++-- python/tests/test_Cluster.py | 18 ++ python/tests/test_ClusterVector.py | 2 +- python/tests/test_Pedestal.py | 5 +- src/NDArray.test.cpp | 34 +++ src/decode.cpp | 52 ++++ src/decode.test.cpp | 82 ++++++ 26 files changed, 597 insertions(+), 280 deletions(-) create mode 100644 docs/src/NumpyArrays.rst diff --git a/RELEASE.md b/RELEASE.md index 256cbb63..6cf4c316 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -35,6 +35,14 @@ ### API Changes: +- The C++ ``NDArray`` view constructor and ``NDArray::copy_from()`` take + ``NDView``. Mutable views still convert implicitly, so + existing callers are unaffected, and read-only views can now be copied. +- Added C++ overloads of ``adc_sar_05_06_07_08decode64to16``, + ``adc_sar_05_decode64to16`` and ``adc_sar_04_decode64to16`` that take the + packed samples as ``NDView``. Words are assembled with + ``memcpy``, so the buffer does not need to be 8-byte aligned. The existing + ``NDView`` overloads are unchanged. - ``NDArray`` ``+ - * /`` with a scalar now returns a lazy expression instead of an ``NDArray`` and no longer converts the scalar to the element type. - ``FastPedestal`` variance is now a private ``double`` intermediate. Removed @@ -80,7 +88,21 @@ - ``RawMasterFile::rois()`` always returns a list of rois (no optional). Per default it returns a list of one ROI element spawing the entire detector - ``TimingMode::Auto`` changed to ``TimingMode::AUTO_TIMING``, ``TimingMode::Trigger`` changed to ``TimingMode::TRIGGER_EXPOSURE`` + ### Bugfixes: +- The Python ``Cluster`` constructors validate that the data array holds + exactly one value per pixel and raise ``ValueError`` otherwise. Previously a + longer array wrote past the end of the cluster data. +- The Python CTB decoding helpers ``adc_sar_*decode64to16``, + ``apply_custom_weights``, ``expand24to32bit``, ``expand4to8bit`` and + ``decode_my302`` validate their input through ``make_view``. They require + C-contiguous arrays of the documented rank and reject other dtypes with + ``TypeError`` instead of converting a copy. The ADC SAR decoders no longer + reinterpret the byte buffer as aligned 64-bit words; a row length that is + not a multiple of 8 bytes raises ``ValueError`` instead of dropping the + trailing bytes. Read-only arrays are accepted where the input is only read. + ``decode_my302`` raises ``ValueError`` instead of ``RuntimeError`` for a + wrong input size. - Mismatched operators inhibited vectorization in gcc of NDArray math operators - ``NDArray``/``NDView`` math operators and expressions were not vectorized for ``uint8_t`` and 64 bit integers, up to 30x slower than a plain loop. @@ -126,6 +148,10 @@ without implicit conversion. Both ``push()`` and ``push_with_threshold()`` validate that frames and thresholds are two-dimensional before constructing views, preventing incorrect results from unsupported array layouts or ranks. +- Python ``ClusterFinder.push_pedestal_frame()`` and ``find_clusters()`` now + require C-contiguous ``uint16`` frames without implicit conversion, raising + ``TypeError`` instead of silently copying and casting the input. The frame + argument can also be passed by keyword as ``frame``. - Fixed a leaked empty ``ClusterVector`` at the end of Python ``ClusterFile`` iteration. Chunk iteration now rejects a zero chunk size. diff --git a/docs/src/NumpyArrays.rst b/docs/src/NumpyArrays.rst new file mode 100644 index 00000000..b837771c --- /dev/null +++ b/docs/src/NumpyArrays.rst @@ -0,0 +1,147 @@ +**************** +Numpy arrays +**************** + +Most Python functions in aare take NumPy arrays as input. The bindings in +``python/src/`` receive them as ``py::array_t`` and wrap the buffer in an +``NDView`` before calling the C++ implementation. No data is copied, so the +C++ code reads and writes the memory that the caller's array owns. This page +explains the choices we make when declaring such an argument and what they +mean for the user. + +The preferred pattern +~~~~~~~~~~~~~~~~~~~~~~~~ + +Declare the argument with only the element type, take it by value, mark it +``noconvert`` in the ``py::arg`` list and let ``make_view`` check the rest: + +.. code-block:: cpp + + m.def( + "push", + [](Pedestal &self, py::array_t frame) { + auto view = make_view_2d(frame); + self.push(view); + }, + py::arg("frame").noconvert()); + +The responsibilities are split between pybind11 and ``make_view``: + +- pybind11 checks that the argument is a NumPy array with exactly the + requested dtype. Anything else fails with a ``TypeError``, and since the + argument is ``noconvert`` pybind11 never tries to build a matching array + from it. +- ``make_view`` (``python/src/np_helper.hpp``) checks that the array has the + expected number of dimensions and is C-contiguous. It raises a + ``ValueError`` with a message that tells the user what to fix, for example + ``Array is not C-contiguous. Use np.ascontiguousarray(arr)``. Writing + functions use ``make_view`` which also rejects read-only arrays. Read-only + input goes through ``make_const_view``. + +The result is a binding that either operates on the caller's memory or +refuses the call. It never silently works on a temporary copy. + +Take the array by value +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``py::array_t`` is a handle to the Python object, not the data. Passing it by +value copies the handle, which increments the reference count of the NumPy +array, and nothing else. The pixel data stays where it is. We therefore +declare the parameter as ``py::array_t frame`` rather than +``py::array_t &frame``: + +- A reference saves one reference-count increment per call, which is not + measurable next to the work done on a frame. +- A by-value parameter is a plain lvalue inside the lambda, so + ``make_view``, which calls ``mutable_data()``, works without further + qualification. A ``const py::array_t &`` does not compile with it. +- It matches the usual pybind11 style for ``py::object`` and its subclasses, + so the binding reads the same as examples in the pybind11 documentation. + +Why not let pybind11 convert +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Without ``noconvert``, pybind11 tries every overload once with conversions +disabled and then once more with conversions enabled. In the second pass a +``py::array_t`` argument accepts anything NumPy can turn into an array of +type ``T``: an array with another dtype, a non-contiguous array, or even a +Python list. pybind11 creates a new array, hands the converted copy to the +binding and drops it when the call returns. For detector data this has +several consequences: + +- The copy costs time and memory, and the user gets no indication that it + happened. For a stack of frames this can be gigabytes. +- Writes are lost. A function that fills or updates its input, for example + a pedestal subtraction in place, writes into the temporary copy. +- Values are cast without warning. ``forcecast`` allows unsafe casts, so a + ``float64`` frame is truncated to ``uint16`` instead of being rejected. +- Overloads stop working as intended. ``apply_calibration`` is registered for + both ``double`` and ``float``. With conversions allowed the first overload + would accept everything by casting and the ``float`` overload would never be + chosen. + +Argument conversion is only worth it for small inputs where convenience beats +control, for example the parameter vector passed to a fit. Even then the +function must not write into the argument. + +.. note:: + + When a binding intentionally lets pybind11 convert, say so with a comment + next to the argument. Otherwise a missing ``noconvert`` looks like an + oversight and the next person to touch the binding will add it. + + .. code-block:: cpp + + // Conversion allowed: small read-only vector, accept lists and any dtype + py::array_t par + +What the template flags mean +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``py::array_t`` takes an optional second template argument. The +flags do two things: they add requirements that the array must fulfil for the +*non-converting* check to pass, and they tell NumPy what to produce when a +conversion *is* allowed. + +``py::array::c_style`` + The array must be C-contiguous. With ``noconvert`` a non-contiguous array + is rejected by pybind11 with a ``TypeError`` listing the accepted + signatures. Without ``noconvert`` pybind11 makes a contiguous copy. + Since ``make_view`` already enforces contiguity and gives a more helpful + ``ValueError``, the flag is redundant in the preferred pattern. It is + still present in some older bindings and is harmless there. + +``py::array::f_style`` + Same as above for Fortran order. ``NDView`` assumes C order, so we do not + use it. + +``py::array::forcecast`` + Allows NumPy to perform unsafe casts, such as ``float64`` to ``uint16``, + when converting. It is the **default** value of the template argument, so + ``py::array_t`` and ``py::array_t`` + are the same type. The flag has no effect together with ``noconvert`` + because no conversion takes place. It only matters when conversions are + allowed, where it decides whether a lossy cast is accepted or the call is + rejected. + +The common spelling ``py::array_t`` +therefore means: when converting, produce a contiguous array of type ``T`` and +cast whatever you were given. Combined with ``noconvert`` it collapses to +"C-contiguous array of exactly type ``T``", which is what the plain +``py::array_t`` plus ``make_view`` already gives us. + +Summary +~~~~~~~~~~~~~~~~~~ + +- Declare the argument as ``py::array_t`` by value and add + ``.noconvert()`` to its ``py::arg``. The handle is reference counted, so + passing it by value does not copy the data. +- Create the view with ``make_view_1d/2d/3d`` or ``make_const_view_*`` and + let it validate dimensions and contiguity. Add an explicit check only when + the binding accepts several shapes, as ``apply_calibration`` does for 2D + and 3D pedestals. +- Return data to Python with ``return_image_data`` or ``return_vector`` so that + the NumPy array owns the C++ allocation. +- Only allow conversion for small, read-only inputs where accepting lists or + other dtypes is a real convenience for the user, and mark it with a comment + in the code. diff --git a/docs/src/index.rst b/docs/src/index.rst index ab22e57d..4f0151b7 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -68,3 +68,4 @@ AARE Philosophy Workflow Tests + NumpyArrays diff --git a/docs/src/pyFit.rst b/docs/src/pyFit.rst index 257f7414..a8375772 100644 --- a/docs/src/pyFit.rst +++ b/docs/src/pyFit.rst @@ -27,4 +27,14 @@ For three-dimensional data, pass an array with shape The result dictionary contains ``par`` and ``chi2``. It also contains ``par_err`` when ``compute_errors`` is enabled. +.. note:: + + The fit works on ``float64`` data. If ``x``, ``y`` or ``y_err`` is not + already a C-contiguous ``float64`` array, the Python bindings convert it + before fitting, which allocates a full copy. Integer or ``float32`` + input therefore works, but for a large three-dimensional ``y`` the + copy can be several times the size of the original array. Pass + ``float64`` arrays to avoid it. A ``float64`` C-contiguous array is used + in place without copying. + .. autofunction:: fit diff --git a/include/aare/NDArray.hpp b/include/aare/NDArray.hpp index 68f88ad6..3a2fbda3 100644 --- a/include/aare/NDArray.hpp +++ b/include/aare/NDArray.hpp @@ -66,14 +66,20 @@ class NDArray : public ArrayExpr, Ndim> { /** * @brief Construct a new NDArray object from a NDView. - * @note The data is copied from the view to the NDArray. + * @note The data is copied from the view to the NDArray. The view is + * only read, so both NDView and NDView are + * accepted. * * @param v view of data to initialize the NDArray with */ - explicit NDArray(const NDView v) : NDArray(v.shape()) { + explicit NDArray(NDView v) : NDArray(v.shape()) { std::copy(v.begin(), v.end(), begin()); } + // Keeps an exact match for mutable views so that they do not pick the + // element-wise ArrayExpr constructor. + explicit NDArray(NDView v) : NDArray(NDView(v)) {} + /** * @brief Construct a new NDArray object from an std::array. */ @@ -149,13 +155,14 @@ class NDArray : public ArrayExpr, Ndim> { /** * @brief Copy data from a view of matching shape into this array without * reallocating. - * @param v view to copy from, must have the same shape as this array + * @param v view to copy from, must have the same shape as this array. + * Accepts NDView and NDView. * @throws std::runtime_error if the shapes differ * * Use this instead of assigning a new NDArray when the buffer needs to be * kept, for example when the array is part of a preallocated pool. */ - void copy_from(const NDView v) { + void copy_from(NDView v) { if (v.shape() != shape_) { throw std::runtime_error(LOCATION + "Shape mismatch in NDArray::copy_from"); diff --git a/include/aare/decode.hpp b/include/aare/decode.hpp index a6efd187..00433109 100644 --- a/include/aare/decode.hpp +++ b/include/aare/decode.hpp @@ -16,6 +16,23 @@ void adc_sar_05_decode64to16(NDView input, void adc_sar_04_decode64to16(NDView input, NDView output); +/** + * @brief Decode packed 64-bit ADC SAR samples stored as raw bytes. + * + * Every 8 bytes of a row form one 64-bit word in native byte order. Words + * are assembled with memcpy, so the buffer does not need to be 8-byte + * aligned. The output must have the same number of rows as the input and + * input.shape(1) / 8 columns. + * @throws std::invalid_argument if the row length is not a multiple of 8 + * bytes or the output shape does not match. + */ +void adc_sar_05_06_07_08decode64to16(NDView input, + NDView output); +void adc_sar_05_decode64to16(NDView input, + NDView output); +void adc_sar_04_decode64to16(NDView input, + NDView output); + /** * @brief Called with a 32 bit unsigned integer, shift by offset * and then return the lower 24 bits as an 32 bit integer diff --git a/python/src/bind_Cluster.hpp b/python/src/bind_Cluster.hpp index 5a749bfc..afa9bfd1 100644 --- a/python/src/bind_Cluster.hpp +++ b/python/src/bind_Cluster.hpp @@ -24,14 +24,21 @@ void define_Cluster(py::module &m, const std::string &typestr) { py::class_>( m, class_name.c_str(), py::buffer_protocol()) + // conversion ok: small input, accept lists and any numeric dtype .def(py::init([](CoordType x, CoordType y, py::array_t data) { - py::buffer_info buf_info = data.request(); + constexpr py::ssize_t n_pixels = ClusterSizeX * ClusterSizeY; + if (data.ndim() != 1 || data.size() != n_pixels) { + throw py::value_error(fmt::format( + "Cluster data must be a 1D array of {} values, got {}D " + "array with {} values", + n_pixels, data.ndim(), data.size())); + } Cluster cluster; cluster.x = x; cluster.y = y; auto r = data.template unchecked<1>(); // no bounds checks - for (py::ssize_t i = 0; i < data.size(); ++i) { + for (py::ssize_t i = 0; i < n_pixels; ++i) { cluster.data[i] = r(i); } return cluster; diff --git a/python/src/bind_ClusterFile.hpp b/python/src/bind_ClusterFile.hpp index 09fc649b..c8544e9d 100644 --- a/python/src/bind_ClusterFile.hpp +++ b/python/src/bind_ClusterFile.hpp @@ -138,9 +138,10 @@ void define_ClusterFile(py::module &m, const std::string &typestr) { "Estimate the number of clusters from the file size. Frame " "headers can make this larger than the actual count.") .def( - "set_noise_map", + "set_noise_map", // conversions ok [](ClusterFile &self, - py::array_t noise_map) { + py::array_t + noise_map) { auto view = make_view_2d(noise_map); self.set_noise_map(view); }, @@ -149,9 +150,10 @@ void define_ClusterFile(py::module &m, const std::string &typestr) { "[y, x]. The map must cover every cluster center coordinate.") .def( - "set_gain_map", + "set_gain_map", // conversions ok [](ClusterFile &self, - py::array_t gain_map) { + py::array_t + gain_map) { auto view = make_view_2d(gain_map); self.set_gain_map(view); }, diff --git a/python/src/bind_ClusterFinder.hpp b/python/src/bind_ClusterFinder.hpp index 2e0bf8b3..661b9d75 100644 --- a/python/src/bind_ClusterFinder.hpp +++ b/python/src/bind_ClusterFinder.hpp @@ -43,12 +43,14 @@ void define_ClusterFinder(py::module &m, const std::string &typestr) { &ClusterFinder::set_nSigma, R"(number of sigma above the pedestal to consider a photon during cluster finding.)") - .def("push_pedestal_frame", - [](ClusterFinder &self, - py::array_t frame) { - auto view = make_view_2d(frame); - self.push_pedestal_frame(view); - }) + .def( + "push_pedestal_frame", + [](ClusterFinder &self, + py::array_t frame) { + auto view = make_view_2d(frame); + self.push_pedestal_frame(view); + }, + py::arg("frame").noconvert()) .def("clear_pedestal", &ClusterFinder::clear_pedestal) .def("update_threshold", @@ -84,7 +86,7 @@ void define_ClusterFinder(py::module &m, const std::string &typestr) { self.find_clusters(view, frame_number); return; }, - py::arg(), py::arg("frame_number") = 0); + py::arg("frame").noconvert(), py::arg("frame_number") = 0); } #pragma GCC diagnostic pop \ No newline at end of file diff --git a/python/src/bind_ClusterFinderMT.hpp b/python/src/bind_ClusterFinderMT.hpp index 37ab8d47..2c166a1a 100644 --- a/python/src/bind_ClusterFinderMT.hpp +++ b/python/src/bind_ClusterFinderMT.hpp @@ -37,12 +37,14 @@ void define_ClusterFinderMT(py::module &m, const std::string &typestr) { py::arg("capacity") = 2048, py::arg("n_threads") = 3, py::arg("queue_depth") = 16, py::arg("min_pedestal_samples") = 1000) - .def("push_pedestal_frame", - [](ClusterFinderMT &self, - py::array_t frame) { - auto view = make_view_2d(frame); - self.push_pedestal_frame(view); - }) + .def( + "push_pedestal_frame", + [](ClusterFinderMT &self, + py::array_t frame) { + auto view = make_view_2d(frame); + self.push_pedestal_frame(view); + }, + py::arg("frame").noconvert()) .def( "find_clusters", [](ClusterFinderMT &self, @@ -50,7 +52,7 @@ void define_ClusterFinderMT(py::module &m, const std::string &typestr) { auto view = make_view_2d(frame); self.find_clusters(view, frame_number); }, - py::arg(), py::arg("frame_number") = 0) + py::arg("frame").noconvert(), py::arg("frame_number") = 0) .def_property_readonly( "cluster_size", [](ClusterFinderMT &self) { diff --git a/python/src/bind_ClusterVector.hpp b/python/src/bind_ClusterVector.hpp index c6888a4a..9b031e5d 100644 --- a/python/src/bind_ClusterVector.hpp +++ b/python/src/bind_ClusterVector.hpp @@ -46,8 +46,7 @@ void define_ClusterVector(py::module &m, const std::string &typestr) { .def( "__call__", - [](ClusterVector &self, - py::array_t mask) { + [](ClusterVector &self, py::array_t mask) { return self(make_view_1d(mask)); }, py::arg("mask").noconvert(), R"doc( diff --git a/python/src/bind_FastPedestal.hpp b/python/src/bind_FastPedestal.hpp index b2b4fb8f..5c278490 100644 --- a/python/src/bind_FastPedestal.hpp +++ b/python/src/bind_FastPedestal.hpp @@ -106,8 +106,7 @@ void define_fast_pedestal_bindings(py::module &m, const std::string &name) { "Return an independent copy of the pedestal and its state.") .def( "push_ema", - [](FastPedestal &pedestal, - py::array_t &frame) { + [](FastPedestal &pedestal, py::array_t frame) { pedestal.push_ema(make_view_2d(frame)); }, py::arg("frame").noconvert(), @@ -115,8 +114,7 @@ void define_fast_pedestal_bindings(py::module &m, const std::string &name) { "already be ready for this update.") .def( "add_init_frame", - [](FastPedestal &pedestal, - py::array_t &frame) { + [](FastPedestal &pedestal, py::array_t frame) { pedestal.add_init_frame(make_view_2d(frame)); }, py::arg("frame").noconvert(), diff --git a/python/src/bind_Interpolator.hpp b/python/src/bind_Interpolator.hpp index a98e539d..3b7a015b 100644 --- a/python/src/bind_Interpolator.hpp +++ b/python/src/bind_Interpolator.hpp @@ -101,9 +101,7 @@ void define_interpolation_bindings(py::module &m) { auto interpolator = py::class_(m, "Interpolator") .def(py::init( - [](py::array_t - etacube, + [](py::array_t etacube, py::array_t xbins, py::array_t ybins, py::array_t ebins) { return Interpolator( @@ -124,9 +122,9 @@ void define_interpolation_bindings(py::module &m) { ebins: bin edges of photon energy )doc", - py::arg("etacube"), - py::arg("xbins"), py::arg("ybins"), - py::arg("ebins")) + py::arg("etacube").noconvert(), + py::arg("xbins").noconvert(), py::arg("ybins").noconvert(), + py::arg("ebins").noconvert()) .def(py::init( [](py::array_t xbins, py::array_t ybins, @@ -146,13 +144,12 @@ void define_interpolation_bindings(py::module &m) { bin edges of etay ebins: bin edges of photon energy - )", py::arg("xbins"), - py::arg("ybins"), py::arg("ebins")) + )", py::arg("xbins").noconvert(), + py::arg("ybins").noconvert(), py::arg("ebins").noconvert()) .def( "rosenblatttransform", [](Interpolator &self, - py::array_t + py::array_t etacube) { return self.rosenblatttransform(make_view_3d(etacube)); }, @@ -162,7 +159,7 @@ void define_interpolation_bindings(py::module &m) { etacube: joint distribution of eta_x, eta_y and photon energy (**Note:** for the joint distribution first dimension is eta_x, second: eta_y, third: energy bins.) )", - py::arg("etacube")) + py::arg("etacube").noconvert()) .def("get_ietax", [](Interpolator &self) { auto *ptr = new NDArray{}; diff --git a/python/src/bind_Pedestal.hpp b/python/src/bind_Pedestal.hpp index 602ee13e..9e8af455 100644 --- a/python/src/bind_Pedestal.hpp +++ b/python/src/bind_Pedestal.hpp @@ -85,11 +85,7 @@ void define_pedestal_bindings(py::module &m, const std::string &name) { // TODO! add push for other data types .def( "push", - [](Pedestal &pedestal, - py::array_t &f) { - if (f.ndim() != 2) { - throw py::value_error("Frame must be 2-dimensional"); - } + [](Pedestal &pedestal, py::array_t f) { auto v = make_view_2d(f); pedestal.push(v); }, @@ -99,15 +95,8 @@ void define_pedestal_bindings(py::module &m, const std::string &name) { "n_samples values per pixel, new values have weight 1 / n_samples.") .def( "push_with_threshold", - [](Pedestal &pedestal, - py::array_t &f, - py::array_t &threshold) { - if (f.ndim() != 2) { - throw py::value_error("Frame must be 2-dimensional"); - } - if (threshold.ndim() != 2) { - throw py::value_error("Threshold must be 2-dimensional"); - } + [](Pedestal &pedestal, py::array_t f, + py::array_t threshold) { auto frame_view = make_view_2d(f); auto threshold_view = make_view_2d(threshold); pedestal.push_with_threshold(frame_view, threshold_view); diff --git a/python/src/bind_PedestalTrackingPixelHistogram.hpp b/python/src/bind_PedestalTrackingPixelHistogram.hpp index 35b8b24a..139b58a1 100644 --- a/python/src/bind_PedestalTrackingPixelHistogram.hpp +++ b/python/src/bind_PedestalTrackingPixelHistogram.hpp @@ -50,8 +50,7 @@ void define_pedestal_tracking_pixel_histogram_bindings(py::module &m) { .def( "push_pedestal_no_update", [](PedestalTrackingPixelHistogram &self, - py::array_t - frame) { + py::array_t frame) { auto view = make_view_2d(frame); self.push_pedestal_no_update(view); }, @@ -104,8 +103,7 @@ void define_pedestal_tracking_pixel_histogram_bindings(py::module &m) { .def( "fill_async", [](PedestalTrackingPixelHistogram &self, - py::array_t - image) { + py::array_t image) { // Copy the numpy buffer into an owned NDArray while we // still hold the GIL so we don't depend on the array's // backing storage outliving this call. diff --git a/python/src/bind_PixelHistogram.hpp b/python/src/bind_PixelHistogram.hpp index d205bce2..d3a1b462 100644 --- a/python/src/bind_PixelHistogram.hpp +++ b/python/src/bind_PixelHistogram.hpp @@ -46,7 +46,7 @@ void define_pixel_histogram_binding(py::module &m, const char *class_name, .def( "fill_async", - [](Hist &self, py::array_t image) { + [](Hist &self, py::array_t image) { // Copy the numpy buffer into an owned NDArray while we // still hold the GIL so we don't depend on the array's // backing storage outliving this call. diff --git a/python/src/bind_calibration.hpp b/python/src/bind_calibration.hpp index c9479e8a..6efa87bc 100644 --- a/python/src/bind_calibration.hpp +++ b/python/src/bind_calibration.hpp @@ -11,38 +11,34 @@ namespace py = pybind11; template -py::array_t pybind_apply_calibration( - py::array_t data, - py::array_t pedestal, - py::array_t - calibration, - int n_threads = 4) { +py::array_t +pybind_apply_calibration(py::array_t data, + py::array_t pedestal, + py::array_t calibration, int n_threads = 4) { auto data_span = make_view_3d(data); // data is always 3D /* No pointer is passed, so NumPy will allocate the buffer */ auto result = py::array_t(data_span.shape()); auto res = make_view_3d(result); - if (data.ndim() == 3 && pedestal.ndim() == 3 && calibration.ndim() == 3) { + if (pedestal.ndim() == 3 && calibration.ndim() == 3) { auto ped = make_view_3d(pedestal); auto cal = make_view_3d(calibration); aare::apply_calibration(res, data_span, ped, cal, n_threads); - } else if (data.ndim() == 3 && pedestal.ndim() == 2 && - calibration.ndim() == 2) { + } else if (pedestal.ndim() == 2 && calibration.ndim() == 2) { auto ped = make_view_2d(pedestal); auto cal = make_view_2d(calibration); aare::apply_calibration(res, data_span, ped, cal, n_threads); } else { throw std::runtime_error( - "Invalid number of dimensions for data, pedestal or calibration"); + "Invalid number of dimensions for pedestal or calibration"); } return result; } -py::array_t pybind_count_switching_pixels( - py::array_t data, - ssize_t n_threads = 4) { +py::array_t pybind_count_switching_pixels(py::array_t data, + ssize_t n_threads = 4) { auto data_span = make_view_3d(data); auto arr = new NDArray{}; @@ -51,9 +47,8 @@ py::array_t pybind_count_switching_pixels( } template -py::array_t pybind_calculate_pedestal( - py::array_t data, - ssize_t n_threads) { +py::array_t pybind_calculate_pedestal(py::array_t data, + ssize_t n_threads) { auto data_span = make_view_3d(data); auto arr = new NDArray{}; @@ -62,9 +57,8 @@ py::array_t pybind_calculate_pedestal( } template -py::array_t pybind_calculate_pedestal_g0( - py::array_t data, - ssize_t n_threads) { +py::array_t pybind_calculate_pedestal_g0(py::array_t data, + ssize_t n_threads) { auto data_span = make_view_3d(data); auto arr = new NDArray{}; diff --git a/python/src/ctb_raw_file.hpp b/python/src/ctb_raw_file.hpp index 5a6953b7..b6338bdd 100644 --- a/python/src/ctb_raw_file.hpp +++ b/python/src/ctb_raw_file.hpp @@ -24,177 +24,119 @@ namespace py = pybind11; using namespace ::aare; +// One binding for the three ADC SAR decoders. Each is registered under +// its original Python name so existing transforms keep working. +template , NDView)> +py::array_t decode64to16(py::array_t input) { + auto bytes = make_const_view_2d(input); + constexpr ssize_t bytes_per_word = sizeof(uint64_t); + auto output = py::array_t( + {bytes.shape(0), bytes.shape(1) / bytes_per_word}); + decode(bytes, make_view_2d(output)); + return output; +} + +constexpr auto decode64to16_doc = + "Decode packed 64-bit ADC samples. Takes a two-dimensional, " + "C-contiguous uint8 array whose row length is a multiple of 8 bytes " + "and returns a uint16 array with one value per 64-bit word."; + void define_ctb_raw_file_io_bindings(py::module &m) { - m.def("adc_sar_05_06_07_08decode64to16", [](py::array_t input) { - if (input.ndim() != 2) { - throw std::runtime_error( - "Only 2D arrays are supported at this moment"); - } + m.def("adc_sar_05_06_07_08decode64to16", + &decode64to16, + py::arg("input").noconvert(), decode64to16_doc); - // Create a 2D output array with the same shape as the input - std::vector shape{input.shape(0), - input.shape(1) / - static_cast(bits_per_byte)}; - py::array_t output(shape); + m.def("adc_sar_05_decode64to16", &decode64to16, + py::arg("input").noconvert(), decode64to16_doc); - // Create a view of the input and output arrays - NDView input_view( - reinterpret_cast(input.mutable_data()), - {output.shape(0), output.shape(1)}); - NDView output_view(output.mutable_data(), - {output.shape(0), output.shape(1)}); + m.def("adc_sar_04_decode64to16", &decode64to16, + py::arg("input").noconvert(), decode64to16_doc); - adc_sar_05_06_07_08decode64to16(input_view, output_view); + m.def( + "apply_custom_weights", + [](py::array_t input, py::array_t weights) { + auto input_view = make_view_1d(input); + auto weights_view = make_view_1d(weights); + auto output = py::array_t(input_view.size()); + apply_custom_weights(input_view, make_view_1d(output), + weights_view); + return output; + }, + py::arg("input").noconvert(), py::arg("weights").noconvert(), + "Apply per-bit weights to every value of a one-dimensional, " + "C-contiguous uint16 array and return a float64 array."); - return output; - }); + m.def( + "expand24to32bit", + [](py::array_t input, uint32_t offset) { + constexpr ssize_t bytes_per_channel = 3; // 24 bit + auto input_view = make_const_view_1d(input); + auto output = + py::array_t(input_view.size() / bytes_per_channel); + aare::expand24to32bit(input_view, make_view_1d(output), + aare::BitOffset(offset)); + return output; + }, + py::arg("input").noconvert(), py::arg("offset"), + "Expand packed 24-bit values from a one-dimensional, C-contiguous " + "uint8 array into a uint32 array."); - m.def("adc_sar_05_decode64to16", [](py::array_t input) { - if (input.ndim() != 2) { - throw std::runtime_error( - "Only 2D arrays are supported at this moment"); - } + m.def( + "expand4to8bit", + [](py::array_t input) { + auto input_view = make_const_view_1d(input); + auto output = py::array_t(input_view.size() * 2); + aare::expand4to8bit(input_view, make_view_1d(output)); + return output; + }, + py::arg("input").noconvert(), + "Expand two 4-bit values per byte of a one-dimensional, C-contiguous " + "uint8 array into a uint8 array of twice the length."); - // Create a 2D output array with the same shape as the input - std::vector shape{input.shape(0), - input.shape(1) / - static_cast(bits_per_byte)}; - py::array_t output(shape); + m.def( + "decode_my302", + [](py::array_t input, uint32_t offset) { + // Physical layout of the chip + constexpr ssize_t channels = 64; + constexpr ssize_t counters = 3; + constexpr ssize_t bytes_per_channel = 3; // 24 bit + constexpr ssize_t n_outputs = 2; - // Create a view of the input and output arrays - NDView input_view( - reinterpret_cast(input.mutable_data()), - {output.shape(0), output.shape(1)}); - NDView output_view(output.mutable_data(), - {output.shape(0), output.shape(1)}); + auto input_view = make_const_view_1d(input); - adc_sar_05_decode64to16(input_view, output_view); + ssize_t expected_size = channels * counters * bytes_per_channel; - return output; - }); + // If we have an offset we need one extra byte per output + aare::BitOffset bitoff(offset); + if (bitoff.value()) + expected_size += n_outputs; - m.def("adc_sar_04_decode64to16", [](py::array_t input) { - if (input.ndim() != 2) { - throw std::runtime_error( - "Only 2D arrays are supported at this moment"); - } + if (input_view.size() != expected_size) { + throw py::value_error( + fmt::format("{} Expected an input size of {} bytes. Called " + "with input size of {}", + LOCATION, expected_size, input_view.size())); + } - // Create a 2D output array with the same shape as the input - std::vector shape{input.shape(0), - input.shape(1) / - static_cast(bits_per_byte)}; - py::array_t output(shape); + auto output = py::array_t(channels * counters); + auto output_view = make_view_1d(output); - // Create a view of the input and output arrays - NDView input_view( - reinterpret_cast(input.mutable_data()), - {output.shape(0), output.shape(1)}); - NDView output_view(output.mutable_data(), - {output.shape(0), output.shape(1)}); + const ssize_t in_step = input_view.size() / n_outputs; + const ssize_t out_step = output_view.size() / n_outputs; + for (ssize_t i = 0; i != n_outputs; ++i) { + NDView in_part( + input_view.data() + in_step * i, {in_step}); + NDView out_part(output_view.data() + out_step * i, + {out_step}); + aare::expand24to32bit(in_part, out_part, bitoff); + } - adc_sar_04_decode64to16(input_view, output_view); - - return output; - }); - - m.def("apply_custom_weights", - [](py::array_t - &input, - py::array_t - &weights) { - // Create new array with same shape as the input array - // (uninitialized values) - py::buffer_info buf = input.request(); - py::array_t output(buf.shape); - - // Use NDViews to call into the C++ library - auto weights_view = make_view_1d(weights); - NDView input_view(input.mutable_data(), - {input.size()}); - NDView output_view(output.mutable_data(), - {output.size()}); - - apply_custom_weights(input_view, output_view, weights_view); - return output; - }); - - m.def("expand24to32bit", - [](py::array_t - &input, - uint32_t offset) { - aare::BitOffset bitoff(offset); - py::buffer_info buf = input.request(); - - constexpr uint32_t bytes_per_channel = 3; // 24 bit - py::array_t output(buf.size / bytes_per_channel); - - NDView input_view(input.mutable_data(), - {input.size()}); - NDView output_view(output.mutable_data(), - {output.size()}); - - aare::expand24to32bit(input_view, output_view, bitoff); - return output; - }); - - m.def("expand4to8bit", - [](py::array_t - &input) { - py::buffer_info buf = input.request(); - - py::array_t output(buf.size * 2); - - NDView input_view(input.mutable_data(), - {input.size()}); - NDView output_view(output.mutable_data(), - {output.size()}); - - aare::expand4to8bit(input_view, output_view); - return output; - }); - - m.def("decode_my302", - [](py::array_t - &input, - uint32_t offset) { - // Physical layout of the chip - constexpr size_t channels = 64; - constexpr size_t counters = 3; - constexpr size_t bytes_per_channel = 3; // 24 bit - constexpr int n_outputs = 2; - - ssize_t expected_size = channels * counters * bytes_per_channel; - - // If whe have an offset we need one extra byte per output - aare::BitOffset bitoff(offset); - if (bitoff.value()) - expected_size += n_outputs; - - if (input.size() != expected_size) { - throw std::runtime_error(fmt::format( - "{} Expected an input size of {} bytes. Called " - "with input size of {}", - LOCATION, expected_size, input.size())); - } - - py::buffer_info buf = input.request(); - py::array_t output(channels * counters); - - for (int i = 0; i != n_outputs; ++i) { - auto step = input.size() / n_outputs; - auto out_step = output.size() / n_outputs; - NDView input_view(input.mutable_data() + step * i, - {input.size() / n_outputs}); - NDView output_view(output.mutable_data() + - out_step * i, - {output.size() / n_outputs}); - - aare::expand24to32bit(input_view, output_view, bitoff); - } - - return output; - }); + return output; + }, + py::arg("input").noconvert(), py::arg("offset"), + "Decode a Mythen 302 readout from a one-dimensional, C-contiguous " + "uint8 array into a uint32 array of 64 channels times 3 counters."); py::class_(m, "CtbRawFile") .def(py::init()) diff --git a/python/src/fit.hpp b/python/src/fit.hpp index 82bff325..1e8d9c39 100644 --- a/python/src/fit.hpp +++ b/python/src/fit.hpp @@ -60,7 +60,8 @@ template void bind_fit_model(py::module &m, const char *name) { .def_property("compute_errors", &FM::compute_errors, &FM::SetComputeErrors) .def( - "__call__", + "__call__", // conversion ok, we want to be able to call with any + // dtype [](const FM & /*self*/, py::array_t x, py::array_t @@ -78,7 +79,7 @@ template void bind_fit_model(py::module &m, const char *name) { }, py::arg("x"), py::arg("par")) .def( - "fit", + "fit", // conversion ok [](const FM &self, py::array_t x, py::array_t y, @@ -163,11 +164,6 @@ fit_dispatch(const aare::FitModel &model, py::array_t>( y_err_obj); - if (y_err.ndim() != 3) { - throw std::runtime_error( - "For 3D input y, y_err must also be 3D."); - } - auto err_out = new NDArray({y.shape(0), y.shape(1), npar}, 0.0); auto y_view_err = make_view_3d(y_err); @@ -208,11 +204,6 @@ fit_dispatch(const aare::FitModel &model, py::array_t>( y_err_obj); - if (y_err.ndim() != 1) { - throw std::runtime_error( - "For 1D input y, y_err must also be 1D."); - } - auto y_view_err = make_view_1d(y_err); result = aare::fit_pixel(model, x_view, y_view, y_view_err); } else { @@ -241,7 +232,8 @@ void define_fit_bindings(py::module &m) { m.def( "fit", - [](py::object model_obj, + [](py::object model_obj, // conversion ok, we want to be able to call + // with any dtype py::array_t x, py::array_t y, py::object y_err_obj, int n_threads) -> py::object { diff --git a/python/src/var_cluster.hpp b/python/src/var_cluster.hpp index 44d70db1..1a3b674c 100644 --- a/python/src/var_cluster.hpp +++ b/python/src/var_cluster.hpp @@ -21,40 +21,42 @@ void define_var_cluster_finder_bindings(py::module &m) { auto *ptr = new NDArray(self.labeled()); return return_image_data(ptr); }) - .def("set_noiseMap", - [](VarClusterFinder &self, - py::array_t - noise_map) { - auto noise_map_span = make_view_2d(noise_map); - self.set_noiseMap(noise_map_span); - }) + .def( + "set_noiseMap", // conversions ok + [](VarClusterFinder &self, + py::array_t + noise_map) { + auto noise_map_span = make_view_2d(noise_map); + self.set_noiseMap(noise_map_span); + }, + py::arg("noise_map")) .def("set_numberOfNeighbours", &VarClusterFinder::set_numberOfNeighbours) .def("set_empty_surroundingPixels", &VarClusterFinder::set_empty_surroundingPixels) .def("set_peripheralThresholdFactor", &VarClusterFinder::set_peripheralThresholdFactor) - .def("find_clusters", - [](VarClusterFinder &self, - py::array_t - img) { - auto view = make_view_2d(img); - self.find_clusters(view); - }) - .def("find_clusters_X", - [](VarClusterFinder &self, - py::array_t - img) { - auto img_span = make_view_2d(img); - self.find_clusters_X(img_span); - }) - .def("single_pass", - [](VarClusterFinder &self, - py::array_t - img) { - auto img_span = make_view_2d(img); - self.single_pass(img_span); - }) + .def( + "find_clusters", + [](VarClusterFinder &self, py::array_t img) { + auto view = make_view_2d(img); + self.find_clusters(view); + }, + py::arg("img").noconvert()) + .def( + "find_clusters_X", + [](VarClusterFinder &self, py::array_t img) { + auto img_span = make_view_2d(img); + self.find_clusters_X(img_span); + }, + py::arg("img").noconvert()) + .def( + "single_pass", + [](VarClusterFinder &self, py::array_t img) { + auto img_span = make_view_2d(img); + self.single_pass(img_span); + }, + py::arg("img").noconvert()) .def("hits", [](VarClusterFinder &self) { auto ptr = new std::vector::Hit>( diff --git a/python/tests/test_Cluster.py b/python/tests/test_Cluster.py index 027cc203..3978ab0c 100644 --- a/python/tests/test_Cluster.py +++ b/python/tests/test_Cluster.py @@ -162,3 +162,21 @@ def test_3x3_reduction(): + + +@pytest.mark.parametrize( + "cluster_type, n_pixels", + [(_aare.Cluster3x3i, 9), (_aare.Cluster2x2i, 4)], +) +def test_cluster_constructor_requires_exact_data_length(cluster_type, n_pixels): + """Data must hold exactly one value per pixel""" + for bad in ( + np.ones(n_pixels - 1, dtype=np.int32), + np.ones(n_pixels + 1, dtype=np.int32), + np.ones((n_pixels, 1), dtype=np.int32), + ): + with pytest.raises(ValueError, match=f"{n_pixels} values"): + cluster_type(0, 0, bad) + + cluster = cluster_type(0, 0, np.arange(n_pixels, dtype=np.int32)) + assert (cluster.data.ravel() == np.arange(n_pixels)).all() diff --git a/python/tests/test_ClusterVector.py b/python/tests/test_ClusterVector.py index dd75366e..b0e9cffa 100644 --- a/python/tests/test_ClusterVector.py +++ b/python/tests/test_ClusterVector.py @@ -146,7 +146,7 @@ def test_masking_requires_c_contiguous_array(): mask = np.array([True, False, True, False], dtype=bool)[::2] assert not mask.flags.c_contiguous - with pytest.raises(TypeError): + with pytest.raises(ValueError, match="C-contiguous"): cv(mask) diff --git a/python/tests/test_Pedestal.py b/python/tests/test_Pedestal.py index 4248d157..d2a3ddb3 100644 --- a/python/tests/test_Pedestal.py +++ b/python/tests/test_Pedestal.py @@ -172,7 +172,7 @@ def test_pedestal_rejects_noncontiguous_inputs(dtype, input_name, layout): invalid = np.ones((4, 6), dtype=input_dtype)[:2, :3][:, ::-1] assert not invalid.flags.c_contiguous - with pytest.raises(TypeError): + with pytest.raises(ValueError, match="C-contiguous"): if input_name == "push_frame": pedestal.push(invalid) elif input_name == "threshold_frame": @@ -192,9 +192,8 @@ def test_pedestal_rejects_inputs_with_wrong_ndim(dtype, input_name, shape): threshold = np.full((2, 3), 10, dtype=dtype) input_dtype = dtype if input_name == "threshold" else np.uint16 invalid = np.ones(shape, dtype=input_dtype) - name = "Threshold" if input_name == "threshold" else "Frame" - with pytest.raises(ValueError, match=f"{name} must be 2-dimensional"): + with pytest.raises(ValueError): if input_name == "push_frame": pedestal.push(invalid) elif input_name == "threshold_frame": diff --git a/src/NDArray.test.cpp b/src/NDArray.test.cpp index 3657c7b0..872f22cd 100644 --- a/src/NDArray.test.cpp +++ b/src/NDArray.test.cpp @@ -86,6 +86,40 @@ TEST_CASE("A const NDArray returns a read-only view") { REQUIRE(view(1, 2) == 1); } +TEST_CASE("NDArray can be constructed from a read-only view") { + NDArray source({2, 3}, 0); + for (ssize_t i = 0; i < source.size(); ++i) + source(i) = static_cast(i); + const auto &const_source = source; + + NDArray from_const(const_source.view()); + NDArray from_mutable(source.view()); + + REQUIRE(from_const.shape() == source.shape()); + REQUIRE(from_mutable.shape() == source.shape()); + for (ssize_t i = 0; i < source.size(); ++i) { + REQUIRE(from_const(i) == source(i)); + REQUIRE(from_mutable(i) == source(i)); + } + + // Independent copy, not a view of the source + source(0) = 42; + REQUIRE(from_const(0) == 0); +} + +TEST_CASE("NDArray::copy_from accepts a read-only view") { + NDArray source({4}, 1.5); + const auto &const_source = source; + + NDArray target({4}, 0.0); + target.copy_from(const_source.view()); + for (ssize_t i = 0; i < target.size(); ++i) + REQUIRE(target(i) == 1.5); + + NDArray wrong_shape({3}, 0.0); + REQUIRE_THROWS(wrong_shape.copy_from(const_source.view())); +} + TEST_CASE("Indexing of a 2D image") { std::array shape{{3, 7}}; NDArray img(shape, 5); diff --git a/src/decode.cpp b/src/decode.cpp index afe261f9..f522f5c1 100644 --- a/src/decode.cpp +++ b/src/decode.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MPL-2.0 #include "aare/decode.hpp" #include +#include #include namespace aare { @@ -102,6 +103,57 @@ void adc_sar_04_decode64to16(NDView input, } } +namespace { + +constexpr ssize_t bytes_per_word = sizeof(uint64_t); + +// Shared loop for the byte-buffer overloads. Words are copied out with +// memcpy so the input does not need to be aligned to 8 bytes. +template +void decode64to16_from_bytes(NDView input, + NDView output) { + if (input.shape(1) % bytes_per_word != 0) { + throw std::invalid_argument( + fmt::format("{} input row length of {} bytes is not a multiple " + "of {}", + LOCATION, input.shape(1), bytes_per_word)); + } + if (input.shape(0) != output.shape(0) || + input.shape(1) / bytes_per_word != output.shape(1)) { + throw std::invalid_argument(fmt::format( + "{} output shape must be ({}, {}) for an input of " + "shape ({}, {})", + LOCATION, input.shape(0), input.shape(1) / bytes_per_word, + output.shape(0), output.shape(1))); + } + + for (ssize_t i = 0; i < output.shape(0); ++i) { + const uint8_t *row = input.data() + i * input.shape(1); + for (ssize_t j = 0; j < output.shape(1); ++j) { + uint64_t word{}; + std::memcpy(&word, row + j * bytes_per_word, sizeof(word)); + output(i, j) = decode_word(word); + } + } +} + +} // namespace + +void adc_sar_05_06_07_08decode64to16(NDView input, + NDView output) { + decode64to16_from_bytes(input, output); +} + +void adc_sar_05_decode64to16(NDView input, + NDView output) { + decode64to16_from_bytes(input, output); +} + +void adc_sar_04_decode64to16(NDView input, + NDView output) { + decode64to16_from_bytes(input, output); +} + double apply_custom_weights(uint16_t input, const NDView weights) { if (weights.size() > 16) { throw std::invalid_argument( diff --git a/src/decode.test.cpp b/src/decode.test.cpp index bab7c38b..09d6c61c 100644 --- a/src/decode.test.cpp +++ b/src/decode.test.cpp @@ -5,6 +5,8 @@ #include #include using Catch::Matchers::WithinAbs; +#include +#include #include using aare::BitOffset; @@ -49,6 +51,86 @@ TEST_CASE("test_adc_sar_05_decode64to16") { CHECK(output == 3072UL); } +TEST_CASE("Decode 64 bit ADC SAR samples from a byte buffer") { + // Two rows of two words each, viewed as raw bytes + aare::NDArray words({2, 2}); + words(0, 0) = 1UL << 29; + words(0, 1) = (1UL << 25) | (1UL << 22); + words(1, 0) = 0; + words(1, 1) = (1UL << 29) | (1UL << 19) | (1UL << 28); + + aare::NDView bytes( + reinterpret_cast(words.data()), {2, 16}); + aare::NDArray out({2, 2}); + aare::adc_sar_05_decode64to16(bytes, out.view()); + + CHECK(out(0, 0) == 1); + CHECK(out(0, 1) == 3072); + CHECK(out(1, 0) == 0); + CHECK(out(1, 1) == 7); + + // Matches the overload that takes the words directly + aare::NDArray reference({2, 2}); + aare::adc_sar_05_decode64to16(words.view(), reference.view()); + for (ssize_t i = 0; i < 2; ++i) { + for (ssize_t j = 0; j < 2; ++j) { + CHECK(out(i, j) == reference(i, j)); + } + } +} + +TEST_CASE("Each byte buffer decoder maps to its own bit order") { + // First two bits of each decoder's bit list + struct Case { + void (*decode)(aare::NDView, + aare::NDView); + int bit0; + int bit1; + }; + const Case cases[] = { + {aare::adc_sar_05_06_07_08decode64to16, 29, 17}, + {aare::adc_sar_05_decode64to16, 29, 19}, + {aare::adc_sar_04_decode64to16, 15, 17}, + }; + + for (const auto &c : cases) { + uint64_t word = (1UL << c.bit0) | (1UL << c.bit1); + std::vector buffer(8); + std::memcpy(buffer.data(), &word, sizeof(word)); + aare::NDView bytes(buffer.data(), {1, 8}); + aare::NDArray out({1, 1}); + c.decode(bytes, out.view()); + CHECK(out(0, 0) == 3); + } +} + +TEST_CASE("Byte buffer decoding does not require an aligned buffer") { + // Place the word three bytes into the buffer + std::vector buffer(3 + 8, 0); + uint64_t word = (1UL << 29) | (1UL << 19); + std::memcpy(buffer.data() + 3, &word, sizeof(word)); + + aare::NDView bytes(buffer.data() + 3, {1, 8}); + aare::NDArray out({1, 1}); + aare::adc_sar_05_decode64to16(bytes, out.view()); + CHECK(out(0, 0) == 3); +} + +TEST_CASE("Byte buffer decoding rejects invalid shapes") { + std::vector buffer(24, 0); + aare::NDArray out({1, 1}); + + // Row length that is not a multiple of 8 + aare::NDView odd_row(buffer.data(), {2, 12}); + REQUIRE_THROWS_AS(aare::adc_sar_05_decode64to16(odd_row, out.view()), + std::invalid_argument); + + // Output shape that does not match rows x (bytes / 8) + aare::NDView two_words(buffer.data(), {1, 16}); + REQUIRE_THROWS_AS(aare::adc_sar_05_decode64to16(two_words, out.view()), + std::invalid_argument); +} + TEST_CASE("test_apply_custom_weights") { uint16_t input = 1;