detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2025-06-03 19:40:40 +02:00
Code Issues Actions Packages Releases Activity

Taking v1 as the first release (#92)

Browse Source 
- file reading
- decoding master file
This commit is contained in:
Erik Fröjdh 2024-11-07 10:14:20 +01:00 committed by GitHub
parent ae71e23dd2
commit d8d1f0c517
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
87 changed files with 9860 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
.github/workflows/build_docs.yml vendored Normal file
View File

@ -0,0 +1,67 @@
name: Build the package using cmake then documentation
on:
workflow_dispatch:
push:
branches:
- v1
permissions:
contents: read
pages: write
id-token: write
jobs:
build:
strategy:
fail-fast: false
matrix:
platform: [ubuntu-latest, ] # macos-12, windows-2019]
python-version: ["3.12",]
runs-on: ${{ matrix.platform }}
# The setup-miniconda action needs this to activate miniconda
defaults:
run:
shell: "bash -l {0}"
steps:
- uses: actions/checkout@v4
- name: Get conda
uses: conda-incubator/setup-miniconda@v3.0.4
with:
python-version: ${{ matrix.python-version }}
channels: conda-forge
- name: Prepare
run: conda install doxygen sphinx=7.1.2 breathe pybind11 sphinx_rtd_theme furo nlohmann_json zeromq fmt numpy
- name: Build library
run: |
mkdir build
cd build
cmake .. -DAARE_SYSTEM_LIBRARIES=ON -DAARE_DOCS=ON
make -j 2
make docs
- name: Upload static files as artifact
id: deployment
uses: actions/upload-pages-artifact@v3
with:
path: build/docs/html/
deploy:
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
runs-on: ubuntu-latest
needs: build
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4

41
.github/workflows/build_pkg.yml vendored Normal file
View File

@ -0,0 +1,41 @@
name: Build packages but don't deploy
on:
workflow_dispatch:
push:
branches:
- v1
#run on PRs as well?
jobs:
build:
strategy:
fail-fast: false
matrix:
platform: [ubuntu-latest, ] # macos-12, windows-2019]
python-version: ["3.12",]
runs-on: ${{ matrix.platform }}
# The setup-miniconda action needs this to activate miniconda
defaults:
run:
shell: "bash -l {0}"
steps:
- uses: actions/checkout@v4
- name: Get conda
uses: conda-incubator/setup-miniconda@v3.0.4
with:
python-version: ${{ matrix.python-version }}
channels: conda-forge
- name: Prepare
run: conda install conda-build conda-verify pytest anaconda-client
- name: Build
env:
CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }}
run: conda build conda-recipe

40
.github/workflows/deploy.yml vendored Normal file
View File

@ -0,0 +1,40 @@
name: Deploy to slsdetectorgroup conda channel
on:
workflow_dispatch
jobs:
build:
strategy:
fail-fast: false
matrix:
platform: [ubuntu-latest, ] # macos-12, windows-2019]
python-version: ["3.12",]
runs-on: ${{ matrix.platform }}
# The setup-miniconda action needs this to activate miniconda
defaults:
run:
shell: "bash -l {0}"
steps:
- uses: actions/checkout@v4
- name: Get conda
uses: conda-incubator/setup-miniconda@v3.0.4
with:
python-version: ${{ matrix.python-version }}
channels: conda-forge
- name: Prepare
run: conda install conda-build conda-verify pytest anaconda-client
- name: Enable upload
run: conda config --set anaconda_upload yes
- name: Build
env:
CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }}
run: conda build conda-recipe --user slsdetectorgroup --token ${CONDA_TOKEN}

382
CMakeLists.txt Normal file
View File

@ -0,0 +1,382 @@
cmake_minimum_required(VERSION 3.14)
project(aare
VERSION 1.0.0
DESCRIPTION "Data processing library for PSI detectors"
HOMEPAGE_URL "https://github.com/slsdetectorgroup/aare"
LANGUAGES C CXX
)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
if (${CMAKE_VERSION} VERSION_GREATER "3.24")
cmake_policy(SET CMP0135 NEW) #Fetch content download timestamp
endif()
cmake_policy(SET CMP0079 NEW)
include(GNUInstallDirs)
include(FetchContent)
#Set default build type if none was specified
include(cmake/helpers.cmake)
default_build_type("Release")
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
# General options
option(AARE_PYTHON_BINDINGS "Build python bindings" ON)
option(AARE_TESTS "Build tests" OFF)
option(AARE_EXAMPLES "Build examples" OFF)
option(AARE_IN_GITHUB_ACTIONS "Running in Github Actions" OFF)
option(AARE_DOCS "Build documentation" OFF)
option(AARE_VERBOSE "Verbose output" OFF)
# Configure which of the dependencies to use FetchContent for
option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON)
option(AARE_FETCH_PYBIND11 "Use FetchContent to download pybind11" ON)
option(AARE_FETCH_CATCH "Use FetchContent to download catch2" ON)
option(AARE_FETCH_JSON "Use FetchContent to download nlohmann::json" ON)
option(AARE_FETCH_ZMQ "Use FetchContent to download libzmq" ON)
#Convenience option to use system libraries only (no FetchContent)
option(AARE_SYSTEM_LIBRARIES "Use system libraries" OFF)
if(AARE_SYSTEM_LIBRARIES)
message(STATUS "Build using system libraries")
set(AARE_FETCH_FMT OFF CACHE BOOL "Disabled FetchContent for FMT" FORCE)
set(AARE_FETCH_PYBIND11 OFF CACHE BOOL "Disabled FetchContent for pybind11" FORCE)
set(AARE_FETCH_CATCH OFF CACHE BOOL "Disabled FetchContent for catch2" FORCE)
set(AARE_FETCH_JSON OFF CACHE BOOL "Disabled FetchContent for nlohmann::json" FORCE)
set(AARE_FETCH_ZMQ OFF CACHE BOOL "Disabled FetchContent for libzmq" FORCE)
endif()
if(AARE_VERBOSE)
add_compile_definitions(AARE_VERBOSE)
endif()
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
if(AARE_FETCH_ZMQ)
# Fetchcontent_Declare is deprecated need to find a way to update this
# for now setting the policy to old is enough
if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.30")
cmake_policy(SET CMP0169 OLD)
endif()
FetchContent_Declare(
libzmq
GIT_REPOSITORY https://github.com/zeromq/libzmq.git
GIT_TAG v4.3.4
)
# Disable unwanted options from libzmq
set(BUILD_TESTS OFF CACHE BOOL "Switch off libzmq test build")
set(BUILD_SHARED OFF CACHE BOOL "Switch off libzmq shared libs")
set(WITH_PERF_TOOL OFF CACHE BOOL "")
set(ENABLE_CPACK OFF CACHE BOOL "")
set(ENABLE_CLANG OFF CACHE BOOL "")
set(ENABLE_CURVE OFF CACHE BOOL "")
set(ENABLE_DRAFTS OFF CACHE BOOL "")
# TODO! Verify that this is what we want to do in aare
# Using GetProperties and Populate to be able to exclude zmq
# from install (not possible with FetchContent_MakeAvailable(libzmq))
FetchContent_GetProperties(libzmq)
if(NOT libzmq_POPULATED)
FetchContent_Populate(libzmq)
add_subdirectory(${libzmq_SOURCE_DIR} ${libzmq_BINARY_DIR} EXCLUDE_FROM_ALL)
endif()
else()
find_package(ZeroMQ 4 REQUIRED)
endif()
if (AARE_FETCH_FMT)
set(FMT_TEST OFF CACHE INTERNAL "disabling fmt tests")
FetchContent_Declare(
fmt
GIT_REPOSITORY https://github.com/fmtlib/fmt.git
GIT_TAG 10.2.1
GIT_PROGRESS TRUE
USES_TERMINAL_DOWNLOAD TRUE
)
FetchContent_MakeAvailable(fmt)
set_property(TARGET fmt PROPERTY POSITION_INDEPENDENT_CODE ON)
else()
find_package(fmt 6 REQUIRED)
endif()
if (AARE_FETCH_JSON)
FetchContent_Declare(
json
URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
)
FetchContent_MakeAvailable(json)
else()
find_package(nlohmann_json 3.11.3 REQUIRED)
endif()
include(GNUInstallDirs)
# If conda build, always set lib dir to 'lib'
if($ENV{CONDA_BUILD})
set(CMAKE_INSTALL_LIBDIR "lib")
endif()
# Set lower / upper case project names
string(TOUPPER "${PROJECT_NAME}" PROJECT_NAME_UPPER)
string(TOLOWER "${PROJECT_NAME}" PROJECT_NAME_LOWER)
# Set targets export name (used by slsDetectorPackage and dependencies)
set(TARGETS_EXPORT_NAME "${PROJECT_NAME_LOWER}-targets")
set(namespace "aare::")
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
# Check if project is being used directly or via add_subdirectory
set(AARE_MASTER_PROJECT OFF)
if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
set(AARE_MASTER_PROJECT ON)
endif()
add_library(aare_compiler_flags INTERFACE)
target_compile_features(aare_compiler_flags INTERFACE cxx_std_17)
if(AARE_PYTHON_BINDINGS)
add_subdirectory(python)
endif()
#################
# MSVC specific #
#################
if(MSVC)
add_compile_definitions(AARE_MSVC)
if(CMAKE_BUILD_TYPE STREQUAL "Release")
message(STATUS "Release build")
target_compile_options(aare_compiler_flags INTERFACE /O2)
else()
message(STATUS "Debug build")
target_compile_options(
aare_compiler_flags
INTERFACE
/Od
/Zi
/MDd
/D_ITERATOR_DEBUG_LEVEL=2
)
target_link_options(
aare_compiler_flags
INTERFACE
/DEBUG:FULL
)
endif()
target_compile_options(
aare_compiler_flags
INTERFACE
/w # disable warnings
)
else()
######################
# GCC/Clang specific #
######################
if(CMAKE_BUILD_TYPE STREQUAL "Release")
message(STATUS "Release build")
target_compile_options(aare_compiler_flags INTERFACE -O3)
else()
message(STATUS "Debug build")
target_compile_options(
aare_compiler_flags
INTERFACE
-Og
-ggdb3
)
endif()
# Common flags for GCC and Clang
target_compile_options(
aare_compiler_flags
INTERFACE
-Wall
-Wextra
-pedantic
-Wshadow
-Wnon-virtual-dtor
-Woverloaded-virtual
-Wdouble-promotion
-Wformat=2
-Wredundant-decls
-Wvla
-Wdouble-promotion
-Werror=return-type #important can cause segfault in optimzed builds
)
endif() #GCC/Clang specific
if(AARE_TESTS)
enable_testing()
add_subdirectory(tests)
endif()
###------------------------------------------------------------------------------MAIN LIBRARY
###------------------------------------------------------------------------------------------
set(PUBLICHEADERS
include/aare/ClusterFinder.hpp
include/aare/CtbRawFile.hpp
include/aare/defs.hpp
include/aare/Dtype.hpp
include/aare/File.hpp
include/aare/FileInterface.hpp
include/aare/RawMasterFile.hpp
include/aare/Frame.hpp
include/aare/NDArray.hpp
include/aare/NDView.hpp
include/aare/NumpyFile.hpp
include/aare/NumpyHelpers.hpp
include/aare/Pedestal.hpp
include/aare/PixelMap.hpp
include/aare/RawFile.hpp
include/aare/SubFile.hpp
include/aare/VarClusterFinder.hpp
)
set(SourceFiles
${CMAKE_CURRENT_SOURCE_DIR}/src/defs.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/CtbRawFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/SubFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp
)
add_library(aare_core STATIC ${SourceFiles})
target_include_directories(aare_core PUBLIC
"$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>"
"$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
)
target_link_libraries(aare_core PUBLIC fmt::fmt PRIVATE aare_compiler_flags nlohmann_json::nlohmann_json)
set_target_properties(aare_core PROPERTIES
ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
PUBLIC_HEADER "${PUBLICHEADERS}"
)
if (AARE_PYTHON_BINDINGS)
set_property(TARGET aare_core PROPERTY POSITION_INDEPENDENT_CODE ON)
endif()
if(AARE_TESTS)
set(TestSources
${CMAKE_CURRENT_SOURCE_DIR}/src/defs.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NDArray.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NDView.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFinder.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/Pedestal.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.test.cpp
)
target_sources(tests PRIVATE ${TestSources} )
endif()
###------------------------------------------------------------------------------------------
###------------------------------------------------------------------------------------------
if(AARE_MASTER_PROJECT)
install(TARGETS aare_core aare_compiler_flags
EXPORT "${TARGETS_EXPORT_NAME}"
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/aare
)
endif()
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(CMAKE_INSTALL_RPATH $ORIGIN)
set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
# #Overall target to link to when using the library
# add_library(aare INTERFACE)
# target_link_libraries(aare INTERFACE aare_core aare_compiler_flags)
# target_include_directories(aare INTERFACE
# $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
# $<INSTALL_INTERFACE:include>
# )
# add_subdirectory(examples)
if(AARE_DOCS)
add_subdirectory(docs)
endif()
# custom target to run check formatting with clang-format
add_custom_target(
check-format
COMMAND find \( -name "*.cpp" -o -name "*.hpp" \) -not -path "./build/*" | xargs -I {} -n 1 -P 10 bash -c "clang-format -Werror -style=\"file:.clang-format\" {} | diff {} -"
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "Checking code formatting with clang-format"
VERBATIM
)
add_custom_target(
format-files
COMMAND find \( -name "*.cpp" -o -name "*.hpp" \) -not -path "./build/*" | xargs -I {} -n 1 -P 10 bash -c "clang-format -i -style=\"file:.clang-format\" {}"
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "Formatting with clang-format"
VERBATIM
)
if (AARE_IN_GITHUB_ACTIONS)
message(STATUS "Running in Github Actions")
set(CLANG_TIDY_COMMAND "clang-tidy-17")
else()
set(CLANG_TIDY_COMMAND "clang-tidy")
endif()
add_custom_target(
clang-tidy
COMMAND find \( -path "./src/*" -a -not -path "./src/python/*" -a \( -name "*.cpp" -not -name "*.test.cpp"\) \) -not -name "CircularFifo.hpp" -not -name "ProducerConsumerQueue.hpp" -not -name "VariableSizeClusterFinder.hpp" | xargs -I {} -n 1 -P 10 bash -c "${CLANG_TIDY_COMMAND} --config-file=.clang-tidy -p build {}"
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "linting with clang-tidy"
VERBATIM
)
# if(AARE_MASTER_PROJECT)
# set(CMAKE_INSTALL_DIR "share/cmake/${PROJECT_NAME}")
# set(PROJECT_LIBRARIES slsSupportShared slsDetectorShared slsReceiverShared)
# include(cmake/package_config.cmake)
# endif()

69
README.md
View File

@ -1,2 +1,71 @@
# aare
Data analysis library for PSI hybrid detectors
## Build and install
Prerequisites
- cmake >= 3.14
- C++17 compiler (gcc >= 8)
- python >= 3.10
### Development install (for Python)
```bash
git clone git@github.com:slsdetectorgroup/aare.git --branch=v1 #or using http...
mkdir build
cd build
#configure using cmake
cmake ../aare
#build (replace 4 with the number of threads you want to use)
make -j4
```
Now you can use the Python module from your build directory
```python
import aare
f = aare.File('Some/File/I/Want_to_open_master_0.json')
```
To run form other folders either add the path to your conda environment using conda-build or add it to your PYTHONPATH
### Install using conda/mamba
```bash
#enable your env first!
conda install aare=2024.10.29.dev0 -c slsdetectorgroup
```
### Install to a custom location and use in your project
Working example in: https://github.com/slsdetectorgroup/aare-examples
```bash
#build and install aare
git clone git@github.com:slsdetectorgroup/aare.git --branch=v1 #or using http...
mkdir build
cd build
#configure using cmake
cmake ../aare -DCMAKE_INSTALL_PREFIX=/where/to/put/aare
#build (replace 4 with the number of threads you want to use)
make -j4
#install
make install
#Now configure your project
cmake .. -DCMAKE_PREFIX_PATH=SOME_PATH
```
### Local build of conda pkgs
```bash
conda build . --variants="{python: [3.11, 3.12, 3.13]}"
```

11
cmake/FindSphinx.cmake Normal file
View File

@ -0,0 +1,11 @@
#Look for an executable called sphinx-build
find_program(SPHINX_EXECUTABLE
NAMES sphinx-build sphinx-build-3.6
DOC "Path to sphinx-build executable")
include(FindPackageHandleStandardArgs)
#Handle standard arguments to find_package like REQUIRED and QUIET
find_package_handle_standard_args(Sphinx
"Failed to find sphinx-build executable"
SPHINX_EXECUTABLE)

6
cmake/helpers.cmake Normal file
View File

@ -0,0 +1,6 @@
function(default_build_type val)
if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
message(STATUS "No build type selected, default to Release")
set(CMAKE_BUILD_TYPE ${val} CACHE STRING "Build type (default ${val})" FORCE)
endif()
endfunction()

35
cmake/package_config.cmake Normal file
View File

@ -0,0 +1,35 @@
# This cmake code creates the configuration that is found and used by
# find_package() of another cmake project
# get lower and upper case project name for the configuration files
# configure and install the configuration files
include(CMakePackageConfigHelpers)
configure_package_config_file(
"${CMAKE_SOURCE_DIR}/cmake/project-config.cmake.in"
"${PROJECT_BINARY_DIR}/${PROJECT_NAME_LOWER}-config.cmake"
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME_LOWER}
PATH_VARS CMAKE_INSTALL_DIR)
write_basic_package_version_file(
"${PROJECT_BINARY_DIR}/${PROJECT_NAME_LOWER}-config-version.cmake"
VERSION ${PROJECT_VERSION}
COMPATIBILITY SameMajorVersion
)
install(FILES
"${PROJECT_BINARY_DIR}/${PROJECT_NAME_LOWER}-config.cmake"
"${PROJECT_BINARY_DIR}/${PROJECT_NAME_LOWER}-config-version.cmake"
COMPONENT devel
DESTINATION ${CMAKE_INSTALL_DIR}
)
if (PROJECT_LIBRARIES OR PROJECT_STATIC_LIBRARIES)
install(
EXPORT "${TARGETS_EXPORT_NAME}"
FILE ${PROJECT_NAME_LOWER}-targets.cmake
DESTINATION ${CMAKE_INSTALL_DIR}
)
endif ()

26
cmake/project-config.cmake.in Normal file
View File

@ -0,0 +1,26 @@
# Config file for @PROJECT_NAME_LOWER@
#
# It defines the following variables:
#
# @PROJECT_NAME_UPPER@_INCLUDE_DIRS - include directory
# @PROJECT_NAME_UPPER@_LIBRARIES - all dynamic libraries
# @PROJECT_NAME_UPPER@_STATIC_LIBRARIES - all static libraries
@PACKAGE_INIT@
include(CMakeFindDependencyMacro)
set(SLS_USE_HDF5 "@SLS_USE_HDF5@")
find_dependency(Threads)
# Add optional dependencies here
if (SLS_USE_HDF5)
find_dependency(HDF5)
endif ()
set_and_check(@PROJECT_NAME_UPPER@_CMAKE_INCLUDE_DIRS "@PACKAGE_CMAKE_INSTALL_DIR@")
include("${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake")
check_required_components("@PROJECT_NAME@")

28
conda-recipe/conda_build_config.yaml Normal file
View File

@ -0,0 +1,28 @@
python:
- 3.11
- 3.11
- 3.11
- 3.12
- 3.12
- 3.12
- 3.13
numpy:
- 1.26
- 2.0
- 2.1
- 1.26
- 2.0
- 2.1
- 2.1
zip_keys:
- python
- numpy
pin_run_as_build:
numpy: x.x
python: x.x

51
conda-recipe/meta.yaml Normal file
View File

@ -0,0 +1,51 @@
package:
name: aare
version: 2024.11.07.dev0 #TODO! how to not duplicate this?
source:
path: ..
build:
number: 0
script:
- unset CMAKE_GENERATOR && {{ PYTHON }} -m pip install . -vv # [not win]
- {{ PYTHON }} -m pip install . -vv # [win]
requirements:
build:
- python {{python}}
- numpy {{ numpy }}
- {{ compiler('cxx') }}
host:
- cmake
- ninja
- python {{python}}
- numpy {{ numpy }}
- pip
- scikit-build-core
- pybind11 >=2.13.0
- fmt
- zeromq
- nlohmann_json
- catch2
run:
- python {{python}}
- numpy {{ numpy }}
test:
imports:
- aare
# requires:
# - pytest
# source_files:
# - tests
# commands:
# - pytest tests
about:
summary: An example project built with pybind11 and scikit-build.
# license_file: LICENSE

68
docs/CMakeLists.txt Normal file
View File

@ -0,0 +1,68 @@
find_package(Doxygen REQUIRED)
find_package(Sphinx REQUIRED)
#Doxygen
set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in)
set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
#Sphinx
set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src)
set(SPHINX_BUILD ${CMAKE_CURRENT_BINARY_DIR})
set(SPHINX_SOURCE_FILES
src/index.rst
src/NDArray.rst
src/NDView.rst
src/File.rst
src/Frame.rst
src/Dtype.rst
src/ClusterFinder.rst
src/Pedestal.rst
src/RawFile.rst
src/RawMasterFile.rst
src/VarClusterFinder.rst
src/pyVarClusterFinder.rst
src/pyFile.rst
src/pyCtbRawFile.rst
src/pyRawMasterFile.rst
)
foreach(filename ${SPHINX_SOURCE_FILES})
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${filename}
"${SPHINX_BUILD}/${filename}")
endforeach(filename ${SPHINX_SOURCE_FILES})
configure_file(
"${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in"
"${SPHINX_BUILD}/conf.py"
@ONLY
)
configure_file(
"${CMAKE_CURRENT_SOURCE_DIR}/static/extra.css"
"${SPHINX_BUILD}/static/css/extra.css"
@ONLY
)
add_custom_target(
docs
COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT}
COMMAND ${SPHINX_EXECUTABLE} -a -b html
-Dbreathe_projects.aare=${CMAKE_CURRENT_BINARY_DIR}/xml
-c "${SPHINX_BUILD}"
${SPHINX_BUILD}/src
${SPHINX_BUILD}/html
COMMENT "Generating documentation with Sphinx"
)
add_custom_target(
rst
COMMAND ${SPHINX_EXECUTABLE} -a -b html
-Dbreathe_projects.aare=${CMAKE_CURRENT_BINARY_DIR}/xml
-c "${SPHINX_BUILD}"
${SPHINX_BUILD}/src
${SPHINX_BUILD}/html
COMMENT "Generating documentation with Sphinx"
)

1917
docs/Doxyfile.in Normal file
View File

File diff suppressed because it is too large Load Diff

64
docs/conf.py.in Normal file
View File

@ -0,0 +1,64 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# http://www.sphinx-doc.org/en/master/config
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('..'))
print(sys.path)
# -- Project information -----------------------------------------------------
project = 'aare'
copyright = '2024, CPS Detector Group'
author = 'CPS Detector Group'
version = '@PROJECT_VERSION@'
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['breathe',
'sphinx_rtd_theme',
'sphinx.ext.autodoc',
'sphinx.ext.napoleon',
]
breathe_default_project = "aare"
napoleon_use_ivar = True
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = "furo"
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['static']
def setup(app):
app.add_css_file('css/extra.css') # may also be an URL

7
docs/src/ClusterFinder.rst Normal file
View File

@ -0,0 +1,7 @@
ClusterFinder
=============
.. doxygenclass:: aare::ClusterFinder
:members:
:undoc-members:

7
docs/src/Dtype.rst Normal file
View File

@ -0,0 +1,7 @@
Dtype
=============
.. doxygenclass:: aare::Dtype
:members:
:undoc-members:

8
docs/src/File.rst Normal file
View File

@ -0,0 +1,8 @@
File
=============
.. doxygenclass:: aare::File
:members:
:undoc-members:
:private-members:

8
docs/src/Frame.rst Normal file
View File

@ -0,0 +1,8 @@
Frame
=============
.. doxygenclass:: aare::Frame
:members:
:undoc-members:
:private-members:

7
docs/src/NDArray.rst Normal file
View File

@ -0,0 +1,7 @@
NDArray
=============
.. doxygenclass:: aare::NDArray
:members:
:undoc-members:

7
docs/src/NDView.rst Normal file
View File

@ -0,0 +1,7 @@
NDView
=============
.. doxygenclass:: aare::NDView
:members:
:undoc-members:

8
docs/src/Pedestal.rst Normal file
View File

@ -0,0 +1,8 @@
Pedestal
=============
.. doxygenclass:: aare::Pedestal
:members:
:undoc-members:
:private-members:

8
docs/src/RawFile.rst Normal file
View File

@ -0,0 +1,8 @@
RawFile
===============
.. doxygenclass:: aare::RawFile
:members:
:undoc-members:
:private-members:

14
docs/src/RawMasterFile.rst Normal file
View File

@ -0,0 +1,14 @@
RawMasterFile
===============
.. doxygenclass:: aare::RawMasterFile
:members:
:undoc-members:
:private-members:
.. doxygenclass:: aare::RawFileNameComponents
:members:
:undoc-members:
:private-members:

7
docs/src/VarClusterFinder.rst Normal file
View File

@ -0,0 +1,7 @@
VarClusterFinder
====================
.. doxygenclass:: aare::VarClusterFinder
:members:
:undoc-members:

31
docs/src/index.rst Normal file
View File

@ -0,0 +1,31 @@
AARE
==============================================
.. note ::
Hello
.. toctree::
:caption: C++ API
:maxdepth: 1
NDArray
NDView
Frame
File
Dtype
ClusterFinder
Pedestal
RawFile
RawMasterFile
VarClusterFinder
.. toctree::
:caption: Python API
:maxdepth: 1
pyFile
pyCtbRawFile
pyRawMasterFile
pyVarClusterFinder

11
docs/src/pyCtbRawFile.rst Normal file
View File

@ -0,0 +1,11 @@
CtbRawFile
============
.. py:currentmodule:: aare
.. autoclass:: CtbRawFile
:members:
:undoc-members:
:show-inheritance:
:inherited-members:

11
docs/src/pyFile.rst Normal file
View File

@ -0,0 +1,11 @@
File
========
.. py:currentmodule:: aare
.. autoclass:: File
:members:
:undoc-members:
:show-inheritance:
:inherited-members:

10
docs/src/pyRawMasterFile.rst Normal file
View File

@ -0,0 +1,10 @@
RawMasterFile
===================
.. py:currentmodule:: aare
.. autoclass:: RawMasterFile
:members:
:undoc-members:
:show-inheritance:
:inherited-members:

10
docs/src/pyVarClusterFinder.rst Normal file
View File

@ -0,0 +1,10 @@
VarClusterFinder
===================
.. py:currentmodule:: aare
.. autoclass:: VarClusterFinder
:members:
:undoc-members:
:show-inheritance:
:inherited-members:

4
docs/static/extra.css vendored Normal file
View File

@ -0,0 +1,4 @@
/* override table no-wrap */
.wy-table-responsive table td, .wy-table-responsive table th {
white-space: normal;
}

148
include/aare/ClusterFileV2.hpp Normal file
View File

@ -0,0 +1,148 @@
#pragma once
#include "aare/core/defs.hpp"
#include <filesystem>
#include <string>
#include <fmt/format.h>
namespace aare {
struct ClusterHeader {
int32_t frame_number;
int32_t n_clusters;
std::string to_string() const {
return "frame_number: " + std::to_string(frame_number) + ", n_clusters: " + std::to_string(n_clusters);
}
};
struct ClusterV2_ {
int16_t x;
int16_t y;
std::array<int32_t, 9> data;
std::string to_string(bool detailed = false) const {
if (detailed) {
std::string data_str = "[";
for (auto &d : data) {
data_str += std::to_string(d) + ", ";
}
data_str += "]";
return "x: " + std::to_string(x) + ", y: " + std::to_string(y) + ", data: " + data_str;
}
return "x: " + std::to_string(x) + ", y: " + std::to_string(y);
}
};
struct ClusterV2 {
ClusterV2_ cluster;
int32_t frame_number;
std::string to_string() const {
return "frame_number: " + std::to_string(frame_number) + ", " + cluster.to_string();
}
};
/**
* @brief
* important not: fp always points to the clusters header and does not point to individual clusters
*
*/
class ClusterFileV2 {
std::filesystem::path m_fpath;
std::string m_mode;
FILE *fp{nullptr};
void check_open(){
if (!fp)
throw std::runtime_error(fmt::format("File: {} not open", m_fpath.string()));
}
public:
ClusterFileV2(std::filesystem::path const &fpath, std::string const &mode): m_fpath(fpath), m_mode(mode) {
if (m_mode != "r" && m_mode != "w")
throw std::invalid_argument("mode must be 'r' or 'w'");
if (m_mode == "r" && !std::filesystem::exists(m_fpath))
throw std::invalid_argument("File does not exist");
if (mode == "r") {
fp = fopen(fpath.string().c_str(), "rb");
} else if (mode == "w") {
if (std::filesystem::exists(fpath)) {
fp = fopen(fpath.string().c_str(), "r+b");
} else {
fp = fopen(fpath.string().c_str(), "wb");
}
}
if (fp == nullptr) {
throw std::runtime_error("Failed to open file");
}
}
~ClusterFileV2() { close(); }
std::vector<ClusterV2> read() {
check_open();
ClusterHeader header;
fread(&header, sizeof(ClusterHeader), 1, fp);
std::vector<ClusterV2_> clusters_(header.n_clusters);
fread(clusters_.data(), sizeof(ClusterV2_), header.n_clusters, fp);
std::vector<ClusterV2> clusters;
for (auto &c : clusters_) {
ClusterV2 cluster;
cluster.cluster = std::move(c);
cluster.frame_number = header.frame_number;
clusters.push_back(cluster);
}
return clusters;
}
std::vector<std::vector<ClusterV2>> read(int n_frames) {
std::vector<std::vector<ClusterV2>> clusters;
for (int i = 0; i < n_frames; i++) {
clusters.push_back(read());
}
return clusters;
}
size_t write(std::vector<ClusterV2> const &clusters) {
check_open();
if (m_mode != "w")
throw std::runtime_error("File not opened in write mode");
if (clusters.empty())
return 0;
ClusterHeader header;
header.frame_number = clusters[0].frame_number;
header.n_clusters = clusters.size();
fwrite(&header, sizeof(ClusterHeader), 1, fp);
for (auto &c : clusters) {
fwrite(&c.cluster, sizeof(ClusterV2_), 1, fp);
}
return clusters.size();
}
size_t write(std::vector<std::vector<ClusterV2>> const &clusters) {
check_open();
if (m_mode != "w")
throw std::runtime_error("File not opened in write mode");
size_t n_clusters = 0;
for (auto &c : clusters) {
n_clusters += write(c);
}
return n_clusters;
}
int seek_to_begin() { return fseek(fp, 0, SEEK_SET); }
int seek_to_end() { return fseek(fp, 0, SEEK_END); }
int32_t frame_number() {
auto pos = ftell(fp);
ClusterHeader header;
fread(&header, sizeof(ClusterHeader), 1, fp);
fseek(fp, pos, SEEK_SET);
return header.frame_number;
}
void close() {
if (fp) {
fclose(fp);
fp = nullptr;
}
}
};
} // namespace aare

259
include/aare/ClusterFinder.hpp Normal file
View File

@ -0,0 +1,259 @@
#pragma once
#include "aare/Dtype.hpp"
#include "aare/NDArray.hpp"
#include "aare/NDView.hpp"
#include "aare/Pedestal.hpp"
#include "aare/defs.hpp"
#include <cstddef>
namespace aare {
/** enum to define the event types */
enum eventType {
PEDESTAL, /** pedestal */
NEIGHBOUR, /** neighbour i.e. below threshold, but in the cluster of a
photon */
PHOTON, /** photon i.e. above threshold */
PHOTON_MAX, /** maximum of a cluster satisfying the photon conditions */
NEGATIVE_PEDESTAL, /** negative value, will not be accounted for as pedestal
in order to avoid drift of the pedestal towards
negative values */
UNDEFINED_EVENT = -1 /** undefined */
};
template <typename FRAME_TYPE = uint16_t, typename PEDESTAL_TYPE = double>
class ClusterFinder {
Shape<2> m_image_size;
const int m_cluster_sizeX;
const int m_cluster_sizeY;
const double m_threshold;
const double m_nSigma;
const double c2;
const double c3;
Pedestal<PEDESTAL_TYPE> m_pedestal;
public:
ClusterFinder(Shape<2> image_size, Shape<2>cluster_size, double nSigma = 5.0,
double threshold = 0.0)
: m_image_size(image_size), m_cluster_sizeX(cluster_size[0]), m_cluster_sizeY(cluster_size[1]),
m_threshold(threshold), m_nSigma(nSigma),
c2(sqrt((m_cluster_sizeY + 1) / 2 * (m_cluster_sizeX + 1) / 2)),
c3(sqrt(m_cluster_sizeX * m_cluster_sizeY)),
m_pedestal(image_size[0], image_size[1]) {
// c2 = sqrt((cluster_sizeY + 1) / 2 * (cluster_sizeX + 1) / 2);
// c3 = sqrt(cluster_sizeX * cluster_sizeY);
};
void push_pedestal_frame(NDView<FRAME_TYPE, 2> frame) {
m_pedestal.push(frame);
}
NDArray<PEDESTAL_TYPE, 2> pedestal() {
return m_pedestal.mean();
}
std::vector<Cluster>
find_clusters_without_threshold(NDView<FRAME_TYPE, 2> frame,
// Pedestal<PEDESTAL_TYPE> &pedestal,
bool late_update = false) {
struct pedestal_update {
int x;
int y;
FRAME_TYPE value;
};
std::vector<pedestal_update> pedestal_updates;
std::vector<Cluster> clusters;
std::vector<std::vector<eventType>> eventMask;
for (int i = 0; i < frame.shape(0); i++) {
eventMask.push_back(std::vector<eventType>(frame.shape(1)));
}
long double val;
long double max;
for (int iy = 0; iy < frame.shape(0); iy++) {
for (int ix = 0; ix < frame.shape(1); ix++) {
// initialize max and total
max = std::numeric_limits<FRAME_TYPE>::min();
long double total = 0;
eventMask[iy][ix] = PEDESTAL;
for (short ir = -(m_cluster_sizeY / 2);
ir < (m_cluster_sizeY / 2) + 1; ir++) {
for (short ic = -(m_cluster_sizeX / 2);
ic < (m_cluster_sizeX / 2) + 1; ic++) {
if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
iy + ir >= 0 && iy + ir < frame.shape(0)) {
val = frame(iy + ir, ix + ic) -
m_pedestal.mean(iy + ir, ix + ic);
total += val;
if (val > max) {
max = val;
}
}
}
}
auto rms = m_pedestal.std(iy, ix);
if (frame(iy, ix) - m_pedestal.mean(iy, ix) < -m_nSigma * rms) {
eventMask[iy][ix] = NEGATIVE_PEDESTAL;
continue;
} else if (max > m_nSigma * rms) {
eventMask[iy][ix] = PHOTON;
} else if (total > c3 * m_nSigma * rms) {
eventMask[iy][ix] = PHOTON;
} else {
if (late_update) {
pedestal_updates.push_back({ix, iy, frame(iy, ix)});
} else {
m_pedestal.push(iy, ix, frame(iy, ix));
}
continue;
}
if (eventMask[iy][ix] == PHOTON &&
(frame(iy, ix) - m_pedestal.mean(iy, ix)) >= max) {
eventMask[iy][ix] = PHOTON_MAX;
Cluster cluster(m_cluster_sizeX, m_cluster_sizeY,
Dtype(typeid(PEDESTAL_TYPE)));
cluster.x = ix;
cluster.y = iy;
short i = 0;
for (short ir = -(m_cluster_sizeY / 2);
ir < (m_cluster_sizeY / 2) + 1; ir++) {
for (short ic = -(m_cluster_sizeX / 2);
ic < (m_cluster_sizeX / 2) + 1; ic++) {
if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
iy + ir >= 0 && iy + ir < frame.shape(0)) {
PEDESTAL_TYPE tmp =
static_cast<PEDESTAL_TYPE>(
frame(iy + ir, ix + ic)) -
m_pedestal.mean(iy + ir, ix + ic);
cluster.set<PEDESTAL_TYPE>(i, tmp);
i++;
}
}
}
clusters.push_back(cluster);
}
}
}
if (late_update) {
for (auto &update : pedestal_updates) {
m_pedestal.push(update.y, update.x, update.value);
}
}
return clusters;
}
// template <typename FRAME_TYPE, typename PEDESTAL_TYPE>
std::vector<Cluster>
find_clusters_with_threshold(NDView<FRAME_TYPE, 2> frame,
Pedestal<PEDESTAL_TYPE> &pedestal) {
assert(m_threshold > 0);
std::vector<Cluster> clusters;
std::vector<std::vector<eventType>> eventMask;
for (int i = 0; i < frame.shape(0); i++) {
eventMask.push_back(std::vector<eventType>(frame.shape(1)));
}
double tthr, tthr1, tthr2;
NDArray<FRAME_TYPE, 2> rest({frame.shape(0), frame.shape(1)});
NDArray<int, 2> nph({frame.shape(0), frame.shape(1)});
// convert to n photons
// nph = (frame-pedestal.mean()+0.5*m_threshold)/m_threshold; // can be
// optimized with expression templates?
for (int iy = 0; iy < frame.shape(0); iy++) {
for (int ix = 0; ix < frame.shape(1); ix++) {
auto val = frame(iy, ix) - pedestal.mean(iy, ix);
nph(iy, ix) = (val + 0.5 * m_threshold) / m_threshold;
nph(iy, ix) = nph(iy, ix) < 0 ? 0 : nph(iy, ix);
rest(iy, ix) = val - nph(iy, ix) * m_threshold;
}
}
// iterate over frame pixels
for (int iy = 0; iy < frame.shape(0); iy++) {
for (int ix = 0; ix < frame.shape(1); ix++) {
eventMask[iy][ix] = PEDESTAL;
// initialize max and total
FRAME_TYPE max = std::numeric_limits<FRAME_TYPE>::min();
long double total = 0;
if (rest(iy, ix) <= 0.25 * m_threshold) {
pedestal.push(iy, ix, frame(iy, ix));
continue;
}
eventMask[iy][ix] = NEIGHBOUR;
// iterate over cluster pixels around the current pixel (ix,iy)
for (short ir = -(m_cluster_sizeY / 2);
ir < (m_cluster_sizeY / 2) + 1; ir++) {
for (short ic = -(m_cluster_sizeX / 2);
ic < (m_cluster_sizeX / 2) + 1; ic++) {
if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
iy + ir >= 0 && iy + ir < frame.shape(0)) {
auto val = frame(iy + ir, ix + ic) -
pedestal.mean(iy + ir, ix + ic);
total += val;
if (val > max) {
max = val;
}
}
}
}
auto rms = pedestal.std(iy, ix);
if (m_nSigma == 0) {
tthr = m_threshold;
tthr1 = m_threshold;
tthr2 = m_threshold;
} else {
tthr = m_nSigma * rms;
tthr1 = m_nSigma * rms * c3;
tthr2 = m_nSigma * rms * c2;
if (m_threshold > 2 * tthr)
tthr = m_threshold - tthr;
if (m_threshold > 2 * tthr1)
tthr1 = tthr - tthr1;
if (m_threshold > 2 * tthr2)
tthr2 = tthr - tthr2;
}
if (total > tthr1 || max > tthr) {
eventMask[iy][ix] = PHOTON;
nph(iy, ix) += 1;
rest(iy, ix) -= m_threshold;
} else {
pedestal.push(iy, ix, frame(iy, ix));
continue;
}
if (eventMask[iy][ix] == PHOTON &&
frame(iy, ix) - pedestal.mean(iy, ix) >= max) {
eventMask[iy][ix] = PHOTON_MAX;
Cluster cluster(m_cluster_sizeX, m_cluster_sizeY,
Dtype(typeid(FRAME_TYPE)));
cluster.x = ix;
cluster.y = iy;
short i = 0;
for (short ir = -(m_cluster_sizeY / 2);
ir < (m_cluster_sizeY / 2) + 1; ir++) {
for (short ic = -(m_cluster_sizeX / 2);
ic < (m_cluster_sizeX / 2) + 1; ic++) {
if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
iy + ir >= 0 && iy + ir < frame.shape(0)) {
auto tmp = frame(iy + ir, ix + ic) -
pedestal.mean(iy + ir, ix + ic);
cluster.set<FRAME_TYPE>(i, tmp);
i++;
}
}
}
clusters.push_back(cluster);
}
}
}
return clusters;
}
};
} // namespace aare

41
include/aare/CtbRawFile.hpp Normal file
View File

@ -0,0 +1,41 @@
#pragma once
#include "aare/FileInterface.hpp"
#include "aare/RawMasterFile.hpp"
#include "aare/Frame.hpp"
#include <filesystem>
#include <fstream>
namespace aare{
class CtbRawFile{
RawMasterFile m_master;
std::ifstream m_file;
size_t m_current_frame{0};
size_t m_current_subfile{0};
size_t m_num_subfiles{0};
public:
CtbRawFile(const std::filesystem::path &fname);
void read_into(std::byte *image_buf, DetectorHeader* header = nullptr);
void seek(size_t frame_index); //!< seek to the given frame index
size_t tell() const; //!< get the frame index of the file pointer
// in the specific class we can expose more functionality
size_t image_size_in_bytes() const;
size_t frames_in_file() const;
RawMasterFile master() const;
private:
void find_subfiles();
size_t sub_file_index(size_t frame_index) const {
return frame_index / m_master.max_frames_per_file();
}
void open_data_file(size_t subfile_index);
};
}

83
include/aare/Dtype.hpp Normal file
View File

@ -0,0 +1,83 @@
#pragma once
#include <cstdint>
#include <map>
#include <string>
#include <typeinfo>
namespace aare {
// The format descriptor is a single character that specifies the type of the data
// - python documentation: https://docs.python.org/3/c-api/arg.html#numbers
// - py::format_descriptor<T>::format() (in pybind11) does not return the same format as
// written in python.org documentation.
// - numpy also doesn't use the same format. and also numpy associates the format
// with variable bitdepth types. (e.g. long is int64 on linux64 and int32 on win64)
// https://numpy.org/doc/stable/reference/arrays.scalars.html
//
// github issue discussing this:
// https://github.com/pybind/pybind11/issues/1908#issuecomment-658358767
//
// [IN LINUX] the difference is for int64 (long) and uint64 (unsigned long). The format
// descriptor is 'q' and 'Q' respectively and in the documentation it is 'l' and 'k'.
// in practice numpy doesn't seem to care when reading buffer info: the library
// interprets 'q' or 'l' as int64 and 'Q' or 'L' as uint64.
// for this reason we decided to use the same format descriptor as pybind to avoid
// any further discrepancies.
// in the following order:
// int8, uint8, int16, uint16, int32, uint32, int64, uint64, float, double
const char DTYPE_FORMAT_DSC[] = {'b', 'B', 'h', 'H', 'i', 'I', 'q', 'Q', 'f', 'd'};
// on linux64 & apple
const char NUMPY_FORMAT_DSC[] = {'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'f', 'd'};
/**
* @brief enum class to define the endianess of the system
*/
enum class endian {
#ifdef _WIN32
little = 0,
big = 1,
native = little
#else
little = __ORDER_LITTLE_ENDIAN__,
big = __ORDER_BIG_ENDIAN__,
native = __BYTE_ORDER__
#endif
};
/**
* @brief class to define the data type of the pixels
* @note only native endianess is supported
*/
class Dtype {
public:
enum TypeIndex { INT8, UINT8, INT16, UINT16, INT32, UINT32, INT64, UINT64, FLOAT, DOUBLE, ERROR, NONE };
uint8_t bitdepth() const;
size_t bytes() const;
std::string format_descr() const { return std::string(1, DTYPE_FORMAT_DSC[static_cast<int>(m_type)]); }
std::string numpy_descr() const { return std::string(1, NUMPY_FORMAT_DSC[static_cast<int>(m_type)]); }
explicit Dtype(const std::type_info &t);
explicit Dtype(std::string_view sv);
static Dtype from_bitdepth(uint8_t bitdepth);
// not explicit to allow conversions form enum to DType
Dtype(Dtype::TypeIndex ti); // NOLINT
bool operator==(const Dtype &other) const noexcept;
bool operator!=(const Dtype &other) const noexcept;
bool operator==(const std::type_info &t) const;
bool operator!=(const std::type_info &t) const;
// bool operator==(DType::TypeIndex ti) const;
// bool operator!=(DType::TypeIndex ti) const;
std::string to_string() const;
void set_type(Dtype::TypeIndex ti) { m_type = ti; }
private:
TypeIndex m_type{TypeIndex::ERROR};
};
} // namespace aare

63
include/aare/File.hpp Normal file
View File

@ -0,0 +1,63 @@
#pragma once
#include "aare/FileInterface.hpp"
#include <memory>
namespace aare {
/**
* @brief RAII File class for reading, and in the future potentially writing
* image files in various formats. Minimal generic interface. For specail fuctions
* plase use the RawFile or NumpyFile classes directly.
* Wraps FileInterface to abstract the underlying file format
* @note **frame_number** refers the the frame number sent by the detector while **frame_index**
* is the position of the frame in the file
*/
class File {
std::unique_ptr<FileInterface> file_impl;
public:
/**
* @brief Construct a new File object
* @param fname path to the file
* @param mode file mode (r, w, a)
* @param cfg file configuration
* @throws std::runtime_error if the file cannot be opened
* @throws std::invalid_argument if the file mode is not supported
*
*/
File(const std::filesystem::path &fname, const std::string &mode="r", const FileConfig &cfg = {});
/**Since the object is responsible for managing the file we disable copy construction */
File(File const &other) = delete;
/**The same goes for copy assignment */
File& operator=(File const &other) = delete;
File(File &&other) noexcept;
File& operator=(File &&other) noexcept;
~File() = default;
Frame read_frame(); //!< read one frame from the file at the current position
Frame read_frame(size_t frame_index); //!< read one frame at the position given by frame number
std::vector<Frame> read_n(size_t n_frames); //!< read n_frames from the file at the current position
void read_into(std::byte *image_buf);
void read_into(std::byte *image_buf, size_t n_frames);
size_t frame_number(size_t frame_index); //!< get the frame number at the given frame index
size_t bytes_per_frame() const;
size_t pixels_per_frame() const;
size_t bytes_per_pixel() const;
size_t bitdepth() const;
void seek(size_t frame_index); //!< seek to the given frame index
size_t tell() const; //!< get the frame index of the file pointer
size_t total_frames() const;
size_t rows() const;
size_t cols() const;
DetectorType detector_type() const;
};
} // namespace aare

161
include/aare/FileInterface.hpp Normal file
View File

@ -0,0 +1,161 @@
#pragma once
#include "aare/Dtype.hpp"
#include "aare/Frame.hpp"
#include "aare/defs.hpp"
#include <filesystem>
#include <vector>
namespace aare {
/**
* @brief FileConfig structure to store the configuration of a file
* dtype: data type of the file
* rows: number of rows in the file
* cols: number of columns in the file
* geometry: geometry of the file
*/
struct FileConfig {
aare::Dtype dtype{typeid(uint16_t)};
uint64_t rows{};
uint64_t cols{};
bool operator==(const FileConfig &other) const {
return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry &&
detector_type == other.detector_type && max_frames_per_file == other.max_frames_per_file;
}
bool operator!=(const FileConfig &other) const { return !(*this == other); }
// rawfile specific
std::string version{};
xy geometry{1, 1};
DetectorType detector_type{DetectorType::Unknown};
int max_frames_per_file{};
size_t total_frames{};
std::string to_string() const {
return "{ dtype: " + dtype.to_string() + ", rows: " + std::to_string(rows) + ", cols: " + std::to_string(cols) +
", geometry: " + geometry.to_string() + ", detector_type: " + ToString(detector_type) +
", max_frames_per_file: " + std::to_string(max_frames_per_file) +
", total_frames: " + std::to_string(total_frames) + " }";
}
};
/**
* @brief FileInterface class to define the interface for file operations
* @note parent class for NumpyFile and RawFile
* @note all functions are pure virtual and must be implemented by the derived classes
*/
class FileInterface {
public:
/**
* @brief one frame from the file at the current position
* @return Frame
*/
virtual Frame read_frame() = 0;
/**
* @brief read one frame from the file at the given frame number
* @param frame_number frame number to read
* @return frame
*/
virtual Frame read_frame(size_t frame_number) = 0;
/**
* @brief read n_frames from the file at the current position
* @param n_frames number of frames to read
* @return vector of frames
*/
virtual std::vector<Frame> read_n(size_t n_frames) = 0; // Is this the right interface?
/**
* @brief read one frame from the file at the current position and store it in the provided buffer
* @param image_buf buffer to store the frame
* @return void
*/
virtual void read_into(std::byte *image_buf) = 0;
/**
* @brief read n_frames from the file at the current position and store them in the provided buffer
* @param image_buf buffer to store the frames
* @param n_frames number of frames to read
* @return void
*/
virtual void read_into(std::byte *image_buf, size_t n_frames) = 0;
/**
* @brief get the frame number at the given frame index
* @param frame_index index of the frame
* @return frame number
*/
virtual size_t frame_number(size_t frame_index) = 0;
/**
* @brief get the size of one frame in bytes
* @return size of one frame
*/
virtual size_t bytes_per_frame() = 0;
/**
* @brief get the number of pixels in one frame
* @return number of pixels in one frame
*/
virtual size_t pixels_per_frame() = 0;
/**
* @brief seek to the given frame number
* @param frame_number frame number to seek to
* @return void
*/
virtual void seek(size_t frame_number) = 0;
/**
* @brief get the current position of the file pointer
* @return current position of the file pointer
*/
virtual size_t tell() = 0;
/**
* @brief get the total number of frames in the file
* @return total number of frames in the file
*/
virtual size_t total_frames() const = 0;
/**
* @brief get the number of rows in the file
* @return number of rows in the file
*/
virtual size_t rows() const = 0;
/**
* @brief get the number of columns in the file
* @return number of columns in the file
*/
virtual size_t cols() const = 0;
/**
* @brief get the bitdepth of the file
* @return bitdepth of the file
*/
virtual size_t bitdepth() const = 0;
virtual DetectorType detector_type() const = 0;
// function to query the data type of the file
/*virtual DataType dtype = 0; */
virtual ~FileInterface() = default;
protected:
std::string m_mode{};
// std::filesystem::path m_fname{};
// std::filesystem::path m_base_path{};
// std::string m_base_name{}, m_ext{};
// int m_findex{};
// size_t m_total_frames{};
// size_t max_frames_per_file{};
// std::string version{};
// DetectorType m_type{DetectorType::Unknown};
// size_t m_rows{};
// size_t m_cols{};
// size_t m_bitdepth{};
// size_t current_frame{};
};
} // namespace aare

124
include/aare/Frame.hpp Normal file
View File

@ -0,0 +1,124 @@
#pragma once
#include "aare/Dtype.hpp"
#include "aare/NDArray.hpp"
#include "aare/defs.hpp"
#include <cstddef>
#include <cstdint>
#include <memory>
#include <vector>
namespace aare {
/**
* @brief Frame class to represent a single frame of data. Not much more than a
* pointer and some info. Limited interface to accept frames from many sources.
*/
class Frame {
uint32_t m_rows;
uint32_t m_cols;
Dtype m_dtype;
std::byte *m_data;
//TODO! Add frame number?
public:
/**
* @brief Construct a new Frame
* @param rows number of rows
* @param cols number of columns
* @param dtype data type of the pixels
* @note the data is initialized to zero
*/
Frame(uint32_t rows, uint32_t cols, Dtype dtype);
/**
* @brief Construct a new Frame
* @param bytes pointer to the data to be copied into the frame
* @param rows number of rows
* @param cols number of columns
* @param dtype data type of the pixels
*/
Frame(const std::byte *bytes, uint32_t rows, uint32_t cols, Dtype dtype);
~Frame(){ delete[] m_data; };
/** @warning Copy is disabled to ensure performance when passing
* frames around. Can discuss enabling it.
*
*/
Frame &operator=(const Frame &other) = delete;
Frame(const Frame &other) = delete;
// enable move
Frame &operator=(Frame &&other) noexcept;
Frame(Frame &&other) noexcept;
Frame clone() const; //<- Explicit copy
uint32_t rows() const;
uint32_t cols() const;
size_t bitdepth() const;
Dtype dtype() const;
uint64_t size() const;
size_t bytes() const;
std::byte *data() const;
/**
* @brief Get the pointer to the pixel at the given row and column
* @param row row index
* @param col column index
* @return pointer to the pixel
* @warning The user should cast the pointer to the appropriate type. Think
* twice if this is the function you want to use.
*/
std::byte *pixel_ptr(uint32_t row, uint32_t col) const;
/**
* @brief Set the pixel at the given row and column to the given value
* @tparam T type of the value
* @param row row index
* @param col column index
* @param data value to set
*/
template <typename T> void set(uint32_t row, uint32_t col, T data) {
assert(sizeof(T) == m_dtype.bytes());
if (row >= m_rows || col >= m_cols) {
throw std::out_of_range("Invalid row or column index");
}
std::memcpy(m_data + (row * m_cols + col) * m_dtype.bytes(), &data,
m_dtype.bytes());
}
template <typename T> T get(uint32_t row, uint32_t col) {
assert(sizeof(T) == m_dtype.bytes());
if (row >= m_rows || col >= m_cols) {
throw std::out_of_range("Invalid row or column index");
}
//TODO! add tests then reimplement using pixel_ptr
T data;
std::memcpy(&data, m_data + (row * m_cols + col) * m_dtype.bytes(),
m_dtype.bytes());
return data;
}
/**
* @brief Return an NDView of the frame. This is the preferred way to access
* data in the frame.
*
* @tparam T type of the pixels
* @return NDView<T, 2>
*/
template <typename T> NDView<T, 2> view() {
std::array<int64_t, 2> shape = {static_cast<int64_t>(m_rows),
static_cast<int64_t>(m_cols)};
T *data = reinterpret_cast<T *>(m_data);
return NDView<T, 2>(data, shape);
}
/**
* @brief Copy the frame data into a new NDArray. This is a deep copy.
*/
template <typename T> NDArray<T> image() {
return NDArray<T>(this->view<T>());
}
};
} // namespace aare

380
include/aare/NDArray.hpp Normal file
View File

@ -0,0 +1,380 @@
#pragma once
/*
Container holding image data, or a time series of image data in contigious
memory.
TODO! Add expression templates for operators
*/
#include "aare/NDView.hpp"
#include <algorithm>
#include <array>
#include <cmath>
#include <fmt/format.h>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <numeric>
namespace aare {
template <typename T, int64_t Ndim = 2> class NDArray {
public:
NDArray() : shape_(), strides_(c_strides<Ndim>(shape_)), data_(nullptr){};
explicit NDArray(std::array<int64_t, Ndim> shape)
: shape_(shape), strides_(c_strides<Ndim>(shape_)),
size_(std::accumulate(shape_.begin(), shape_.end(), 1, std::multiplies<>())), data_(new T[size_]){};
NDArray(std::array<int64_t, Ndim> shape, T value) : NDArray(shape) { this->operator=(value); }
/* When constructing from a NDView we need to copy the data since
NDArray expect to own its data, and span is just a view*/
explicit NDArray(NDView<T, Ndim> span) : NDArray(span.shape()) {
std::copy(span.begin(), span.end(), begin());
// fmt::print("NDArray(NDView<T, Ndim> span)\n");
}
// Move constructor
NDArray(NDArray &&other) noexcept
: shape_(other.shape_), strides_(c_strides<Ndim>(shape_)), size_(other.size_), data_(other.data_) {
other.reset();
// fmt::print("NDArray(NDArray &&other)\n");
}
// Copy constructor
NDArray(const NDArray &other)
: shape_(other.shape_), strides_(c_strides<Ndim>(shape_)), size_(other.size_), data_(new T[size_]) {
std::copy(other.data_, other.data_ + size_, data_);
// fmt::print("NDArray(const NDArray &other)\n");
}
~NDArray() { delete[] data_; }
auto begin() { return data_; }
auto end() { return data_ + size_; }
using value_type = T;
NDArray &operator=(NDArray &&other) noexcept; // Move assign
NDArray &operator=(const NDArray &other); // Copy assign
NDArray operator+(const NDArray &other);
NDArray &operator+=(const NDArray &other);
NDArray operator-(const NDArray &other);
NDArray &operator-=(const NDArray &other);
NDArray operator*(const NDArray &other);
NDArray &operator*=(const NDArray &other);
NDArray operator/(const NDArray &other);
// NDArray& operator/=(const NDArray& other);
template <typename V> NDArray &operator/=(const NDArray<V, Ndim> &other) {
// check shape
if (shape_ == other.shape()) {
for (uint32_t i = 0; i < size_; ++i) {
data_[i] /= other(i);
}
return *this;
}
throw(std::runtime_error("Shape of NDArray must match"));
}
NDArray<bool, Ndim> operator>(const NDArray &other);
bool operator==(const NDArray &other) const;
bool operator!=(const NDArray &other) const;
NDArray &operator=(const T & /*value*/);
NDArray &operator+=(const T & /*value*/);
NDArray operator+(const T & /*value*/);
NDArray &operator-=(const T & /*value*/);
NDArray operator-(const T & /*value*/);
NDArray &operator*=(const T & /*value*/);
NDArray operator*(const T & /*value*/);
NDArray &operator/=(const T & /*value*/);
NDArray operator/(const T & /*value*/);
NDArray &operator&=(const T & /*mask*/);
void sqrt() {
for (int i = 0; i < size_; ++i) {
data_[i] = std::sqrt(data_[i]);
}
}
NDArray &operator++(); // pre inc
template <typename... Ix> std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) {
return data_[element_offset(strides_, index...)];
}
template <typename... Ix> std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) const {
return data_[element_offset(strides_, index...)];
}
template <typename... Ix> std::enable_if_t<sizeof...(Ix) == Ndim, T> value(Ix... index) {
return data_[element_offset(strides_, index...)];
}
T &operator()(int i) { return data_[i]; }
const T &operator()(int i) const { return data_[i]; }
T *data() { return data_; }
std::byte *buffer() { return reinterpret_cast<std::byte *>(data_); }
uint64_t size() const { return size_; }
size_t total_bytes() const { return size_ * sizeof(T); }
std::array<int64_t, Ndim> shape() const noexcept { return shape_; }
int64_t shape(int64_t i) const noexcept { return shape_[i]; }
std::array<int64_t, Ndim> strides() const noexcept { return strides_; }
size_t bitdepth() const noexcept { return sizeof(T) * 8; }
std::array<int64_t, Ndim> byte_strides() const noexcept {
auto byte_strides = strides_;
for (auto &val : byte_strides)
val *= sizeof(T);
return byte_strides;
// return strides_;
}
NDView<T, Ndim> span() const { return NDView<T, Ndim>{data_, shape_}; }
void Print();
void Print_all();
void Print_some();
void reset() {
data_ = nullptr;
size_ = 0;
std::fill(shape_.begin(), shape_.end(), 0);
std::fill(strides_.begin(), strides_.end(), 0);
}
private:
std::array<int64_t, Ndim> shape_;
std::array<int64_t, Ndim> strides_;
uint64_t size_{};
T *data_;
};
// Move assign
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(NDArray<T, Ndim> &&other) noexcept {
if (this != &other) {
delete[] data_;
data_ = other.data_;
shape_ = other.shape_;
size_ = other.size_;
strides_ = other.strides_;
other.reset();
}
return *this;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator+(const NDArray &other) {
NDArray result(*this);
result += other;
return result;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator+=(const NDArray<T, Ndim> &other) {
// check shape
if (shape_ == other.shape_) {
for (uint32_t i = 0; i < size_; ++i) {
data_[i] += other.data_[i];
}
return *this;
}
throw(std::runtime_error("Shape of ImageDatas must match"));
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator-(const NDArray &other) {
NDArray result{*this};
result -= other;
return result;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator-=(const NDArray<T, Ndim> &other) {
// check shape
if (shape_ == other.shape_) {
for (uint32_t i = 0; i < size_; ++i) {
data_[i] -= other.data_[i];
}
return *this;
}
throw(std::runtime_error("Shape of ImageDatas must match"));
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator*(const NDArray &other) {
NDArray result = *this;
result *= other;
return result;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator*=(const NDArray<T, Ndim> &other) {
// check shape
if (shape_ == other.shape_) {
for (uint32_t i = 0; i < size_; ++i) {
data_[i] *= other.data_[i];
}
return *this;
}
throw(std::runtime_error("Shape of ImageDatas must match"));
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator/(const NDArray &other) {
NDArray result = *this;
result /= other;
return result;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator&=(const T &mask) {
for (auto it = begin(); it != end(); ++it)
*it &= mask;
return *this;
}
// template <typename T, int64_t Ndim>
// NDArray<T, Ndim>& NDArray<T, Ndim>::operator/=(const NDArray<T, Ndim>&
// other)
// {
// //check shape
// if (shape_ == other.shape_) {
// for (int i = 0; i < size_; ++i) {
// data_[i] /= other.data_[i];
// }
// return *this;
// } else {
// throw(std::runtime_error("Shape of ImageDatas must match"));
// }
// }
template <typename T, int64_t Ndim> NDArray<bool, Ndim> NDArray<T, Ndim>::operator>(const NDArray &other) {
if (shape_ == other.shape_) {
NDArray<bool> result{shape_};
for (int i = 0; i < size_; ++i) {
result(i) = (data_[i] > other.data_[i]);
}
return result;
}
throw(std::runtime_error("Shape of ImageDatas must match"));
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(const NDArray<T, Ndim> &other) {
if (this != &other) {
delete[] data_;
shape_ = other.shape_;
strides_ = other.strides_;
size_ = other.size_;
data_ = new T[size_];
std::copy(other.data_, other.data_ + size_, data_);
}
return *this;
}
template <typename T, int64_t Ndim> bool NDArray<T, Ndim>::operator==(const NDArray<T, Ndim> &other) const {
if (shape_ != other.shape_)
return false;
for (uint32_t i = 0; i != size_; ++i)
if (data_[i] != other.data_[i])
return false;
return true;
}
template <typename T, int64_t Ndim> bool NDArray<T, Ndim>::operator!=(const NDArray<T, Ndim> &other) const {
return !((*this) == other);
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator++() {
for (uint32_t i = 0; i < size_; ++i)
data_[i] += 1;
return *this;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(const T &value) {
std::fill_n(data_, size_, value);
return *this;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator+=(const T &value) {
for (uint32_t i = 0; i < size_; ++i)
data_[i] += value;
return *this;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator+(const T &value) {
NDArray result = *this;
result += value;
return result;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator-=(const T &value) {
for (uint32_t i = 0; i < size_; ++i)
data_[i] -= value;
return *this;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator-(const T &value) {
NDArray result = *this;
result -= value;
return result;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator/=(const T &value) {
for (uint32_t i = 0; i < size_; ++i)
data_[i] /= value;
return *this;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator/(const T &value) {
NDArray result = *this;
result /= value;
return result;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator*=(const T &value) {
for (uint32_t i = 0; i < size_; ++i)
data_[i] *= value;
return *this;
}
template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator*(const T &value) {
NDArray result = *this;
result *= value;
return result;
}
template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print() {
if (shape_[0] < 20 && shape_[1] < 20)
Print_all();
else
Print_some();
}
template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print_all() {
for (auto row = 0; row < shape_[0]; ++row) {
for (auto col = 0; col < shape_[1]; ++col) {
std::cout << std::setw(3);
std::cout << (*this)(row, col) << " ";
}
std::cout << "\n";
}
}
template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print_some() {
for (auto row = 0; row < 5; ++row) {
for (auto col = 0; col < 5; ++col) {
std::cout << std::setw(7);
std::cout << (*this)(row, col) << " ";
}
std::cout << "\n";
}
}
template <typename T, int64_t Ndim> void save(NDArray<T, Ndim> &img, std::string &pathname) {
std::ofstream f;
f.open(pathname, std::ios::binary);
f.write(img.buffer(), img.size() * sizeof(T));
f.close();
}
template <typename T, int64_t Ndim>
NDArray<T, Ndim> load(const std::string &pathname, std::array<int64_t, Ndim> shape) {
NDArray<T, Ndim> img{shape};
std::ifstream f;
f.open(pathname, std::ios::binary);
f.read(img.buffer(), img.size() * sizeof(T));
f.close();
return img;
}
} // namespace aare

159
include/aare/NDView.hpp Normal file
View File

@ -0,0 +1,159 @@
#pragma once
#include <algorithm>
#include <array>
#include <cassert>
#include <cstdint>
#include <functional>
#include <iomanip>
#include <iostream>
#include <numeric>
#include <stdexcept>
#include <vector>
namespace aare {
template <int64_t Ndim> using Shape = std::array<int64_t, Ndim>;
// TODO! fix mismatch between signed and unsigned
template <int64_t Ndim> Shape<Ndim> make_shape(const std::vector<size_t> &shape) {
if (shape.size() != Ndim)
throw std::runtime_error("Shape size mismatch");
Shape<Ndim> arr;
std::copy_n(shape.begin(), Ndim, arr.begin());
return arr;
}
template <int64_t Dim = 0, typename Strides> int64_t element_offset(const Strides & /*unused*/) { return 0; }
template <int64_t Dim = 0, typename Strides, typename... Ix>
int64_t element_offset(const Strides &strides, int64_t i, Ix... index) {
return i * strides[Dim] + element_offset<Dim + 1>(strides, index...);
}
template <int64_t Ndim> std::array<int64_t, Ndim> c_strides(const std::array<int64_t, Ndim> &shape) {
std::array<int64_t, Ndim> strides{};
std::fill(strides.begin(), strides.end(), 1);
for (int64_t i = Ndim - 1; i > 0; --i) {
strides[i - 1] = strides[i] * shape[i];
}
return strides;
}
template <int64_t Ndim> std::array<int64_t, Ndim> make_array(const std::vector<int64_t> &vec) {
assert(vec.size() == Ndim);
std::array<int64_t, Ndim> arr{};
std::copy_n(vec.begin(), Ndim, arr.begin());
return arr;
}
template <typename T, int64_t Ndim = 2> class NDView {
public:
NDView() = default;
~NDView() = default;
NDView(const NDView &) = default;
NDView(NDView &&) = default;
NDView(T *buffer, std::array<int64_t, Ndim> shape)
: buffer_(buffer), strides_(c_strides<Ndim>(shape)), shape_(shape),
size_(std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>())) {}
// NDView(T *buffer, const std::vector<int64_t> &shape)
// : buffer_(buffer), strides_(c_strides<Ndim>(make_array<Ndim>(shape))), shape_(make_array<Ndim>(shape)),
// size_(std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>())) {}
template <typename... Ix> std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) {
return buffer_[element_offset(strides_, index...)];
}
template <typename... Ix> std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) const {
return buffer_[element_offset(strides_, index...)];
}
uint64_t size() const { return size_; }
size_t total_bytes() const { return size_ * sizeof(T); }
std::array<int64_t, Ndim> strides() const noexcept { return strides_; }
T *begin() { return buffer_; }
T *end() { return buffer_ + size_; }
T &operator()(int64_t i) const { return buffer_[i]; }
T &operator[](int64_t i) const { return buffer_[i]; }
bool operator==(const NDView &other) const {
if (size_ != other.size_)
return false;
for (uint64_t i = 0; i != size_; ++i) {
if (buffer_[i] != other.buffer_[i])
return false;
}
return true;
}
NDView &operator+=(const T val) { return elemenwise(val, std::plus<T>()); }
NDView &operator-=(const T val) { return elemenwise(val, std::minus<T>()); }
NDView &operator*=(const T val) { return elemenwise(val, std::multiplies<T>()); }
NDView &operator/=(const T val) { return elemenwise(val, std::divides<T>()); }
NDView &operator/=(const NDView &other) { return elemenwise(other, std::divides<T>()); }
NDView &operator=(const T val) {
for (auto it = begin(); it != end(); ++it)
*it = val;
return *this;
}
NDView &operator=(const NDView &other) {
if (this == &other)
return *this;
shape_ = other.shape_;
strides_ = other.strides_;
size_ = other.size_;
buffer_ = other.buffer_;
return *this;
}
NDView &operator=(NDView &&other) noexcept {
if (this == &other)
return *this;
shape_ = std::move(other.shape_);
strides_ = std::move(other.strides_);
size_ = other.size_;
buffer_ = other.buffer_;
other.buffer_ = nullptr;
return *this;
}
auto &shape() { return shape_; }
auto shape(int64_t i) const { return shape_[i]; }
T *data() { return buffer_; }
void print_all() const;
private:
T *buffer_{nullptr};
std::array<int64_t, Ndim> strides_{};
std::array<int64_t, Ndim> shape_{};
uint64_t size_{};
template <class BinaryOperation> NDView &elemenwise(T val, BinaryOperation op) {
for (uint64_t i = 0; i != size_; ++i) {
buffer_[i] = op(buffer_[i], val);
}
return *this;
}
template <class BinaryOperation> NDView &elemenwise(const NDView &other, BinaryOperation op) {
for (uint64_t i = 0; i != size_; ++i) {
buffer_[i] = op(buffer_[i], other.buffer_[i]);
}
return *this;
}
};
template <typename T, int64_t Ndim> void NDView<T, Ndim>::print_all() const {
for (auto row = 0; row < shape_[0]; ++row) {
for (auto col = 0; col < shape_[1]; ++col) {
std::cout << std::setw(3);
std::cout << (*this)(row, col) << " ";
}
std::cout << "\n";
}
}
} // namespace aare

119
include/aare/NumpyFile.hpp Normal file
View File

@ -0,0 +1,119 @@
#pragma once
#include "aare/Dtype.hpp"
#include "aare/defs.hpp"
#include "aare/FileInterface.hpp"
#include "aare/NumpyHelpers.hpp"
#include <filesystem>
#include <iostream>
#include <numeric>
namespace aare {
/**
* @brief NumpyFile class to read and write numpy files
* @note derived from FileInterface
* @note implements all the pure virtual functions from FileInterface
* @note documentation for the functions can also be found in the FileInterface class
*/
class NumpyFile : public FileInterface {
public:
/**
* @brief NumpyFile constructor
* @param fname path to the numpy file
* @param mode file mode (r, w)
* @param cfg file configuration
*/
explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {});
void write(Frame &frame);
Frame read_frame() override { return get_frame(this->current_frame++); }
Frame read_frame(size_t frame_number) override { return get_frame(frame_number); }
std::vector<Frame> read_n(size_t n_frames) override;
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }
void read_into(std::byte *image_buf, size_t n_frames) override;
size_t frame_number(size_t frame_index) override { return frame_index; };
size_t bytes_per_frame() override;
size_t pixels_per_frame() override;
void seek(size_t frame_number) override { this->current_frame = frame_number; }
size_t tell() override { return this->current_frame; }
size_t total_frames() const override { return m_header.shape[0]; }
size_t rows() const override { return m_header.shape[1]; }
size_t cols() const override { return m_header.shape[2]; }
size_t bitdepth() const override { return m_header.dtype.bitdepth(); }
DetectorType detector_type() const override { return DetectorType::Unknown; }
/**
* @brief get the data type of the numpy file
* @return DType
*/
Dtype dtype() const { return m_header.dtype; }
/**
* @brief get the shape of the numpy file
* @return vector of type size_t
*/
std::vector<size_t> shape() const { return m_header.shape; }
/**
* @brief load the numpy file into an NDArray
* @tparam T data type of the NDArray
* @tparam NDim number of dimensions of the NDArray
* @return NDArray<T, NDim>
*/
template <typename T, size_t NDim> NDArray<T, NDim> load() {
NDArray<T, NDim> arr(make_shape<NDim>(m_header.shape));
if (fseek(fp, static_cast<int64_t>(header_size), SEEK_SET)) {
throw std::runtime_error(LOCATION + "Error seeking to the start of the data");
}
size_t rc = fread(arr.data(), sizeof(T), arr.size(), fp);
if (rc != static_cast<size_t>(arr.size())) {
throw std::runtime_error(LOCATION + "Error reading data from file");
}
return arr;
}
template <typename A, typename TYPENAME, A Ndim> void write(NDView<TYPENAME, Ndim> &frame) {
write_impl(frame.data(), frame.total_bytes());
}
template <typename A, typename TYPENAME, A Ndim> void write(NDArray<TYPENAME, Ndim> &frame) {
write_impl(frame.data(), frame.total_bytes());
}
template <typename A, typename TYPENAME, A Ndim> void write(NDView<TYPENAME, Ndim> &&frame) {
write_impl(frame.data(), frame.total_bytes());
}
template <typename A, typename TYPENAME, A Ndim> void write(NDArray<TYPENAME, Ndim> &&frame) {
write_impl(frame.data(), frame.total_bytes());
}
~NumpyFile() noexcept override;
private:
FILE *fp = nullptr;
size_t initial_header_len = 0;
size_t current_frame{};
uint32_t header_len{};
uint8_t header_len_size{};
size_t header_size{};
NumpyHeader m_header;
uint8_t major_ver_{};
uint8_t minor_ver_{};
size_t m_bytes_per_frame{};
size_t m_pixels_per_frame{};
size_t m_cols;
size_t m_rows;
size_t m_bitdepth;
void load_metadata();
void get_frame_into(size_t /*frame_number*/, std::byte * /*image_buf*/);
Frame get_frame(size_t frame_number);
void write_impl(void *data, uint64_t size);
};
} // namespace aare

55
include/aare/NumpyHelpers.hpp Normal file
View File

@ -0,0 +1,55 @@
#pragma once
#include <algorithm>
#include <array>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <numeric>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>
#include "aare/Dtype.hpp"
#include "aare/defs.hpp"
namespace aare {
struct NumpyHeader {
Dtype dtype{aare::Dtype::ERROR};
bool fortran_order{false};
std::vector<size_t> shape{};
std::string to_string() const;
};
namespace NumpyHelpers {
const constexpr std::array<char, 6> magic_str{'\x93', 'N', 'U', 'M', 'P', 'Y'};
const uint8_t magic_string_length{6};
std::string parse_str(const std::string &in);
/**
Removes leading and trailing whitespaces
*/
std::string trim(const std::string &str);
std::vector<std::string> parse_tuple(std::string in);
bool parse_bool(const std::string &in);
std::string get_value_from_map(const std::string &mapstr);
std::unordered_map<std::string, std::string> parse_dict(std::string in, const std::vector<std::string> &keys);
template <typename T, size_t N> bool in_array(T val, const std::array<T, N> &arr) {
return std::find(std::begin(arr), std::end(arr), val) != std::end(arr);
}
bool is_digits(const std::string &str);
aare::Dtype parse_descr(std::string typestring);
size_t write_header(const std::filesystem::path &fname, const NumpyHeader &header);
size_t write_header(std::ostream &out, const NumpyHeader &header);
} // namespace NumpyHelpers
} // namespace aare

149
include/aare/Pedestal.hpp Normal file
View File

@ -0,0 +1,149 @@
#pragma once
#include "aare/Frame.hpp"
#include "aare/NDArray.hpp"
#include "aare/NDView.hpp"
#include <cstddef>
namespace aare {
/**
* @brief Calculate the pedestal of a series of frames. Can be used as
* standalone but mostly used in the ClusterFinder.
*
* @tparam SUM_TYPE type of the sum
*/
template <typename SUM_TYPE = double> class Pedestal {
uint32_t m_rows;
uint32_t m_cols;
uint32_t m_samples;
NDArray<uint32_t, 2> m_cur_samples;
NDArray<SUM_TYPE, 2> m_sum;
NDArray<SUM_TYPE, 2> m_sum2;
public:
Pedestal(uint32_t rows, uint32_t cols, uint32_t n_samples = 1000)
: m_rows(rows), m_cols(cols), m_samples(n_samples),
m_cur_samples(NDArray<uint32_t, 2>({rows, cols}, 0)),
m_sum(NDArray<SUM_TYPE, 2>({rows, cols})),
m_sum2(NDArray<SUM_TYPE, 2>({rows, cols})) {
assert(rows > 0 && cols > 0 && n_samples > 0);
m_sum = 0;
m_sum2 = 0;
}
~Pedestal() = default;
NDArray<SUM_TYPE, 2> mean() {
NDArray<SUM_TYPE, 2> mean_array({m_rows, m_cols});
for (uint32_t i = 0; i < m_rows * m_cols; i++) {
mean_array(i / m_cols, i % m_cols) = mean(i / m_cols, i % m_cols);
}
return mean_array;
}
SUM_TYPE mean(const uint32_t row, const uint32_t col) const {
if (m_cur_samples(row, col) == 0) {
return 0.0;
}
return m_sum(row, col) / m_cur_samples(row, col);
}
NDArray<SUM_TYPE, 2> variance() {
NDArray<SUM_TYPE, 2> variance_array({m_rows, m_cols});
for (uint32_t i = 0; i < m_rows * m_cols; i++) {
variance_array(i / m_cols, i % m_cols) =
variance(i / m_cols, i % m_cols);
}
return variance_array;
}
SUM_TYPE variance(const uint32_t row, const uint32_t col) const {
if (m_cur_samples(row, col) == 0) {
return 0.0;
}
return m_sum2(row, col) / m_cur_samples(row, col) -
mean(row, col) * mean(row, col);
}
NDArray<SUM_TYPE, 2> std() {
NDArray<SUM_TYPE, 2> standard_deviation_array({m_rows, m_cols});
for (uint32_t i = 0; i < m_rows * m_cols; i++) {
standard_deviation_array(i / m_cols, i % m_cols) =
std(i / m_cols, i % m_cols);
}
return standard_deviation_array;
}
SUM_TYPE std(const uint32_t row, const uint32_t col) const {
return std::sqrt(variance(row, col));
}
void clear() {
for (uint32_t i = 0; i < m_rows * m_cols; i++) {
clear(i / m_cols, i % m_cols);
}
}
void clear(const uint32_t row, const uint32_t col) {
m_sum(row, col) = 0;
m_sum2(row, col) = 0;
m_cur_samples(row, col) = 0;
}
// frame level operations
template <typename T> void push(NDView<T, 2> frame) {
assert(frame.size() == m_rows * m_cols);
// TODO! move away from m_rows, m_cols
if (frame.shape() != std::array<int64_t, 2>{m_rows, m_cols}) {
throw std::runtime_error(
"Frame shape does not match pedestal shape");
}
for (uint32_t row = 0; row < m_rows; row++) {
for (uint32_t col = 0; col < m_cols; col++) {
push<T>(row, col, frame(row, col));
}
}
// // TODO: test the effect of #pragma omp parallel for
// for (uint32_t index = 0; index < m_rows * m_cols; index++) {
// push<T>(index / m_cols, index % m_cols, frame(index));
// }
}
template <typename T> void push(Frame &frame) {
assert(frame.rows() == static_cast<size_t>(m_rows) &&
frame.cols() == static_cast<size_t>(m_cols));
push<T>(frame.view<T>());
}
// getter functions
uint32_t rows() const { return m_rows; }
uint32_t cols() const { return m_cols; }
uint32_t n_samples() const { return m_samples; }
NDArray<uint32_t, 2> cur_samples() const { return m_cur_samples; }
NDArray<SUM_TYPE, 2> get_sum() const { return m_sum; }
NDArray<SUM_TYPE, 2> get_sum2() const { return m_sum2; }
// pixel level operations (should be refactored to allow users to implement
// their own pixel level operations)
template <typename T>
void push(const uint32_t row, const uint32_t col, const T val_) {
SUM_TYPE val = static_cast<SUM_TYPE>(val_);
const uint32_t idx = index(row, col);
if (m_cur_samples(idx) < m_samples) {
m_sum(idx) += val;
m_sum2(idx) += val * val;
m_cur_samples(idx)++;
} else {
m_sum(idx) += val - m_sum(idx) / m_cur_samples(idx);
m_sum2(idx) += val * val - m_sum2(idx) / m_cur_samples(idx);
}
}
uint32_t index(const uint32_t row, const uint32_t col) const {
return row * m_cols + col;
};
};
} // namespace aare

15
include/aare/PixelMap.hpp Normal file
View File

@ -0,0 +1,15 @@
#pragma once
#include "aare/defs.hpp"
#include "aare/NDArray.hpp"
namespace aare {
NDArray<ssize_t, 2> GenerateMoench03PixelMap();
NDArray<ssize_t, 2> GenerateMoench05PixelMap();
NDArray<ssize_t, 2>GenerateMH02SingleCounterPixelMap();
NDArray<ssize_t, 3> GenerateMH02FourCounterPixelMap();
} // namespace aare

110
include/aare/RawFile.hpp Normal file
View File

@ -0,0 +1,110 @@
#pragma once
#include "aare/FileInterface.hpp"
#include "aare/RawMasterFile.hpp"
#include "aare/Frame.hpp"
#include "aare/NDArray.hpp" //for pixel map
#include "aare/SubFile.hpp"
#include <optional>
namespace aare {
struct ModuleConfig {
int module_gap_row{};
int module_gap_col{};
bool operator==(const ModuleConfig &other) const {
if (module_gap_col != other.module_gap_col)
return false;
if (module_gap_row != other.module_gap_row)
return false;
return true;
}
};
/**
* @brief Class to read .raw files. The class will parse the master file
* to find the correct geometry for the frames.
* @note A more generic interface is available in the aare::File class.
* Consider using that unless you need raw file specific functionality.
*/
class RawFile : public FileInterface {
size_t n_subfiles{};
size_t n_subfile_parts{};
std::vector<std::vector<SubFile *>> subfiles;
std::vector<xy> positions;
ModuleConfig cfg{0, 0};
RawMasterFile m_master;
size_t m_current_frame{};
size_t m_rows{};
size_t m_cols{};
public:
/**
* @brief RawFile constructor
* @param fname path to the master file (.json)
* @param mode file mode (only "r" is supported at the moment)
*/
RawFile(const std::filesystem::path &fname, const std::string &mode = "r");
virtual ~RawFile() override;
Frame read_frame() override;
Frame read_frame(size_t frame_number) override;
std::vector<Frame> read_n(size_t n_frames) override;
void read_into(std::byte *image_buf) override;
void read_into(std::byte *image_buf, size_t n_frames) override;
size_t frame_number(size_t frame_index) override;
size_t bytes_per_frame() override;
size_t pixels_per_frame() override;
void seek(size_t frame_index) override;
size_t tell() override;
size_t total_frames() const override;
size_t rows() const override;
size_t cols() const override;
size_t bitdepth() const override;
xy geometry();
DetectorType detector_type() const override;
private:
/**
* @brief check if the file is a master file
* @param fpath path to the file
*/
static bool is_master_file(const std::filesystem::path &fpath);
// TODO! Deal with fast quad and missing files
/**
* @brief read the frame at the given frame index into the image buffer
* @param frame_number frame number to read
* @param image_buf buffer to store the frame
*/
void get_frame_into(size_t frame_index, std::byte *frame_buffer);
/**
* @brief get the frame at the given frame index
* @param frame_number frame number to read
* @return Frame
*/
Frame get_frame(size_t frame_index);
/**
* @brief read the header of the file
* @param fname path to the data subfile
* @return DetectorHeader
*/
static DetectorHeader read_header(const std::filesystem::path &fname);
void find_number_of_subfiles();
void open_subfiles();
void find_geometry();
};
} // namespace aare

131
include/aare/RawMasterFile.hpp Normal file
View File

@ -0,0 +1,131 @@
#pragma once
#include "aare/defs.hpp"
#include <filesystem>
#include <fmt/format.h>
#include <fstream>
#include <optional>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
namespace aare {
/**
* @brief Implementation used in RawMasterFile to parse the file name
*/
class RawFileNameComponents {
std::filesystem::path m_base_path{};
std::string m_base_name{};
std::string m_ext{};
int m_file_index{}; // TODO! is this measurement_index?
public:
RawFileNameComponents(const std::filesystem::path &fname);
/// @brief Get the filename including path of the master file.
/// (i.e. what was passed in to the constructor))
std::filesystem::path master_fname() const;
/// @brief Get the filename including path of the data file.
/// @param mod_id module id run_d[module_id]_f0_0
/// @param file_id file id run_d0_f[file_id]_0
std::filesystem::path data_fname(size_t mod_id, size_t file_id) const;
const std::filesystem::path &base_path() const;
const std::string &base_name() const;
const std::string &ext() const;
int file_index() const;
};
class ScanParameters {
bool m_enabled = false;
std::string m_dac;
int m_start = 0;
int m_stop = 0;
int m_step = 0;
//TODO! add settleTime, requires string to time conversion
public:
ScanParameters(const std::string &par);
ScanParameters() = default;
ScanParameters(const ScanParameters &) = default;
ScanParameters &operator=(const ScanParameters &) = default;
ScanParameters(ScanParameters &&) = default;
int start() const;
int stop() const;
int step() const;
const std::string &dac() const;
bool enabled() const;
};
/**
* @brief Class for parsing a master file either in our .json format or the old
* .raw format
*/
class RawMasterFile {
RawFileNameComponents m_fnc;
std::string m_version;
DetectorType m_type;
TimingMode m_timing_mode;
size_t m_image_size_in_bytes{};
size_t m_frames_in_file{};
size_t m_total_frames_expected{};
size_t m_pixels_y{};
size_t m_pixels_x{};
size_t m_bitdepth{};
xy m_geometry;
size_t m_max_frames_per_file{};
uint32_t m_adc_mask{};
FrameDiscardPolicy m_frame_discard_policy{};
size_t m_frame_padding{};
// TODO! should these be bool?
uint8_t m_analog_flag{};
uint8_t m_digital_flag{};
uint8_t m_transceiver_flag{};
ScanParameters m_scan_parameters;
std::optional<size_t> m_analog_samples;
std::optional<size_t> m_digital_samples;
std::optional<size_t> m_transceiver_samples;
std::optional<size_t> m_number_of_rows;
std::optional<uint8_t> m_quad;
public:
RawMasterFile(const std::filesystem::path &fpath);
std::filesystem::path data_fname(size_t mod_id, size_t file_id) const;
const std::string &version() const; //!< For example "7.2"
const DetectorType &detector_type() const;
const TimingMode &timing_mode() const;
size_t image_size_in_bytes() const;
size_t frames_in_file() const;
size_t pixels_y() const;
size_t pixels_x() const;
size_t max_frames_per_file() const;
size_t bitdepth() const;
size_t frame_padding() const;
const FrameDiscardPolicy &frame_discard_policy() const;
size_t total_frames_expected() const;
xy geometry() const;
std::optional<size_t> analog_samples() const;
std::optional<size_t> digital_samples() const;
std::optional<size_t> transceiver_samples() const;
std::optional<size_t> number_of_rows() const;
std::optional<uint8_t> quad() const;
ScanParameters scan_parameters() const;
private:
void parse_json(const std::filesystem::path &fpath);
void parse_raw(const std::filesystem::path &fpath);
};
} // namespace aare

80
include/aare/SubFile.hpp Normal file
View File

@ -0,0 +1,80 @@
#pragma once
#include "aare/Frame.hpp"
#include "aare/defs.hpp"
#include <cstdint>
#include <filesystem>
#include <map>
#include <optional>
namespace aare {
/**
* @brief Class to read a subfile from a RawFile
*/
class SubFile {
public:
size_t write_part(std::byte *buffer, DetectorHeader header, size_t frame_index);
/**
* @brief SubFile constructor
* @param fname path to the subfile
* @param detector detector type
* @param rows number of rows in the subfile
* @param cols number of columns in the subfile
* @param bitdepth bitdepth of the subfile
* @throws std::invalid_argument if the detector,type pair is not supported
*/
SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
const std::string &mode = "r");
/**
* @brief read the subfile into a buffer
* @param buffer pointer to the buffer to read the data into
* @return number of bytes read
*/
size_t read_impl_normal(std::byte *buffer);
/**
* @brief read the subfile into a buffer with the bytes flipped
* @param buffer pointer to the buffer to read the data into
* @return number of bytes read
*/
template <typename DataType> size_t read_impl_flip(std::byte *buffer);
/**
* @brief read the subfile into a buffer with the bytes reordered
* @param buffer pointer to the buffer to read the data into
* @return number of bytes read
*/
template <typename DataType> size_t read_impl_reorder(std::byte *buffer);
/**
* @brief read the subfile into a buffer with the bytes reordered and flipped
* @param buffer pointer to the buffer to read the data into
* @param frame_number frame number to read
* @return number of bytes read
*/
size_t get_part(std::byte *buffer, size_t frame_index);
size_t frame_number(size_t frame_index);
// TODO: define the inlines as variables and assign them in constructor
inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; }
inline size_t pixels_per_part() const { return m_rows * m_cols; }
~SubFile();
protected:
FILE *fp = nullptr;
size_t m_bitdepth;
std::filesystem::path m_fname;
size_t m_rows{};
size_t m_cols{};
std::string m_mode;
size_t n_frames{};
int m_sub_file_index_{};
DetectorType m_detector_type;
std::optional<NDArray<ssize_t, 2>> pixel_map;
};
} // namespace aare

307
include/aare/VarClusterFinder.hpp Normal file
View File

@ -0,0 +1,307 @@
#pragma once
#include <algorithm>
#include <map>
#include <unordered_map>
#include <vector>
#include "aare/NDArray.hpp"
const int MAX_CLUSTER_SIZE = 200;
namespace aare {
template <typename T> class VarClusterFinder {
public:
struct Hit {
int16_t size{};
int16_t row{};
int16_t col{};
uint16_t reserved{}; // for alignment
T energy{};
T max{};
// std::vector<int16_t> rows{};
// std::vector<int16_t> cols{};
int16_t rows[MAX_CLUSTER_SIZE] = {0};
int16_t cols[MAX_CLUSTER_SIZE] = {0};
double enes[MAX_CLUSTER_SIZE] = {0};
};
private:
const std::array<int64_t, 2> shape_;
NDView<T, 2> original_;
NDArray<int, 2> labeled_;
NDArray<int, 2> peripheral_labeled_;
NDArray<bool, 2> binary_; // over threshold flag
T threshold_;
NDView<T, 2> noiseMap;
bool use_noise_map = false;
int peripheralThresholdFactor_ = 5;
int current_label;
const std::array<int, 4> di{{0, -1, -1, -1}}; // row ### 8-neighbour by scaning from left to right
const std::array<int, 4> dj{{-1, -1, 0, 1}}; // col ### 8-neighbour by scaning from top to bottom
const std::array<int, 8> di_{{0, 0, -1, 1, -1, 1, -1, 1}}; // row
const std::array<int, 8> dj_{{-1, 1, 0, 0, 1, -1, -1, 1}}; // col
std::map<int, int> child; // heirachy: key: child; val: parent
std::unordered_map<int, Hit> h_size;
std::vector<Hit> hits;
// std::vector<std::vector<int16_t>> row
int check_neighbours(int i, int j);
public:
VarClusterFinder(Shape<2> shape, T threshold)
: shape_(shape), labeled_(shape, 0), peripheral_labeled_(shape, 0), binary_(shape), threshold_(threshold) {
hits.reserve(2000);
}
NDArray<int, 2> labeled() { return labeled_; }
void set_noiseMap(NDView<T, 2> noise_map) {
noiseMap = noise_map;
use_noise_map = true;
}
void set_peripheralThresholdFactor(int factor) { peripheralThresholdFactor_ = factor; }
void find_clusters(NDView<T, 2> img);
void find_clusters_X(NDView<T, 2> img);
void rec_FillHit(int clusterIndex, int i, int j);
void single_pass(NDView<T, 2> img);
void first_pass();
void second_pass();
void store_clusters();
std::vector<Hit> steal_hits() {
std::vector<Hit> tmp;
std::swap(tmp, hits);
return tmp;
};
void clear_hits() { hits.clear(); };
void print_connections() {
fmt::print("Connections:\n");
for (auto it = child.begin(); it != child.end(); ++it) {
fmt::print("{} -> {}\n", it->first, it->second);
}
}
size_t total_clusters() const {
// TODO! fix for stealing
return hits.size();
}
private:
void add_link(int from, int to) {
// we want to add key from -> value to
// fmt::print("add_link({},{})\n", from, to);
auto it = child.find(from);
if (it == child.end()) {
child[from] = to;
} else {
// found need to disambiguate
if (it->second == to)
return;
else {
if (it->second > to) {
// child[from] = to;
auto old = it->second;
it->second = to;
add_link(old, to);
} else {
// found value is smaller than what we want to link
add_link(to, it->second);
}
}
}
}
};
template <typename T> int VarClusterFinder<T>::check_neighbours(int i, int j) {
std::vector<int> neighbour_labels;
for (int k = 0; k < 4; ++k) {
const auto row = i + di[k];
const auto col = j + dj[k];
if (row >= 0 && col >= 0 && row < shape_[0] && col < shape_[1]) {
auto tmp = labeled_.value(i + di[k], j + dj[k]);
if (tmp != 0)
neighbour_labels.push_back(tmp);
}
}
if (neighbour_labels.size() == 0) {
return 0;
} else {
// need to sort and add to union field
std::sort(neighbour_labels.rbegin(), neighbour_labels.rend());
auto first = neighbour_labels.begin();
auto last = std::unique(first, neighbour_labels.end());
if (last - first == 1)
return *neighbour_labels.begin();
for (auto current = first; current != last - 1; ++current) {
auto next = current + 1;
add_link(*current, *next);
}
return neighbour_labels.back(); // already sorted
}
}
template <typename T> void VarClusterFinder<T>::find_clusters(NDView<T, 2> img) {
original_ = img;
labeled_ = 0;
peripheral_labeled_ = 0;
current_label = 0;
child.clear();
first_pass();
// print_connections();
second_pass();
store_clusters();
}
template <typename T> void VarClusterFinder<T>::find_clusters_X(NDView<T, 2> img) {
original_ = img;
int clusterIndex = 0;
for (int i = 0; i < shape_[0]; ++i) {
for (int j = 0; j < shape_[1]; ++j) {
if (use_noise_map)
threshold_ = 5 * noiseMap(i, j);
if (original_(i, j) > threshold_) {
// printf("========== Cluster index: %d\n", clusterIndex);
rec_FillHit(clusterIndex, i, j);
clusterIndex++;
}
}
}
for (const auto &h : h_size)
hits.push_back(h.second);
h_size.clear();
}
template <typename T> void VarClusterFinder<T>::rec_FillHit(int clusterIndex, int i, int j) {
// printf("original_(%d, %d)=%f\n", i, j, original_(i,j));
// printf("h_size[%d].size=%d\n", clusterIndex, h_size[clusterIndex].size);
if (h_size[clusterIndex].size < MAX_CLUSTER_SIZE) {
h_size[clusterIndex].rows[h_size[clusterIndex].size] = i;
h_size[clusterIndex].cols[h_size[clusterIndex].size] = j;
h_size[clusterIndex].enes[h_size[clusterIndex].size] = original_(i, j);
}
h_size[clusterIndex].size += 1;
h_size[clusterIndex].energy += original_(i, j);
if (h_size[clusterIndex].max < original_(i, j)) {
h_size[clusterIndex].row = i;
h_size[clusterIndex].col = j;
h_size[clusterIndex].max = original_(i, j);
}
original_(i, j) = 0;
for (int k = 0; k < 8; ++k) { // 8 for 8-neighbour
const auto row = i + di_[k];
const auto col = j + dj_[k];
if (row >= 0 && col >= 0 && row < shape_[0] && col < shape_[1]) {
if (use_noise_map)
threshold_ = peripheralThresholdFactor_ * noiseMap(row, col);
if (original_(row, col) > threshold_) {
rec_FillHit(clusterIndex, row, col);
} else {
// if (h_size[clusterIndex].size < MAX_CLUSTER_SIZE){
// h_size[clusterIndex].size += 1;
// h_size[clusterIndex].rows[h_size[clusterIndex].size] = row;
// h_size[clusterIndex].cols[h_size[clusterIndex].size] = col;
// h_size[clusterIndex].enes[h_size[clusterIndex].size] = original_(row, col);
// }// ? weather to include peripheral pixels
original_(row, col) = 0; // remove peripheral pixels, to avoid potential influence for pedestal updating
}
}
}
}
template <typename T> void VarClusterFinder<T>::single_pass(NDView<T, 2> img) {
original_ = img;
labeled_ = 0;
current_label = 0;
child.clear();
first_pass();
// print_connections();
// second_pass();
// store_clusters();
}
template <typename T> void VarClusterFinder<T>::first_pass() {
for (int i = 0; i < original_.size(); ++i) {
if (use_noise_map)
threshold_ = 5 * noiseMap(i);
binary_(i) = (original_(i) > threshold_);
}
for (int i = 0; i < shape_[0]; ++i) {
for (int j = 0; j < shape_[1]; ++j) {
// do we have someting to process?
if (binary_(i, j)) {
auto tmp = check_neighbours(i, j);
if (tmp != 0) {
labeled_(i, j) = tmp;
} else {
labeled_(i, j) = ++current_label;
}
}
}
}
}
template <typename T> void VarClusterFinder<T>::second_pass() {
for (int64_t i = 0; i != labeled_.size(); ++i) {
auto current_label = labeled_(i);
if (current_label != 0) {
auto it = child.find(current_label);
while (it != child.end()) {
current_label = it->second;
it = child.find(current_label);
// do this once before doing the second pass?
// all values point to the final one...
}
labeled_(i) = current_label;
}
}
}
template <typename T> void VarClusterFinder<T>::store_clusters() {
// Accumulate hit information in a map
// Do we always have monotonic increasing
// labels? Then vector?
// here the translation is label -> Hit
std::unordered_map<int, Hit> h_size;
for (int i = 0; i < shape_[0]; ++i) {
for (int j = 0; j < shape_[1]; ++j) {
if (labeled_(i, j) != 0 || false
// (i-1 >= 0 and labeled_(i-1, j) != 0) or // another circle of peripheral pixels
// (j-1 >= 0 and labeled_(i, j-1) != 0) or
// (i+1 < shape_[0] and labeled_(i+1, j) != 0) or
// (j+1 < shape_[1] and labeled_(i, j+1) != 0)
) {
Hit &record = h_size[labeled_(i, j)];
if (record.size < MAX_CLUSTER_SIZE) {
record.rows[record.size] = i;
record.cols[record.size] = j;
record.enes[record.size] = original_(i, j);
} else {
continue;
}
record.size += 1;
record.energy += original_(i, j);
if (record.max < original_(i, j)) {
record.row = i;
record.col = j;
record.max = original_(i, j);
}
}
}
}
for (const auto &h : h_size)
hits.push_back(h.second);
}
} // namespace aare

197
include/aare/defs.hpp Normal file
View File

@ -0,0 +1,197 @@
#pragma once
#include "aare/Dtype.hpp"
// #include "aare/utils/logger.hpp"
#include <array>
#include <stdexcept>
#include <cassert>
#include <cstdint>
#include <cstring>
#include <string>
#include <string_view>
#include <variant>
#include <vector>
/**
* @brief LOCATION macro to get the current location in the code
*/
#define LOCATION \
std::string(__FILE__) + std::string(":") + std::to_string(__LINE__) + \
":" + std::string(__func__) + ":"
namespace aare {
class Cluster {
public:
int cluster_sizeX;
int cluster_sizeY;
int16_t x;
int16_t y;
Dtype dt;
private:
std::byte *m_data;
public:
Cluster(int cluster_sizeX_, int cluster_sizeY_,
Dtype dt_ = Dtype(typeid(int32_t)))
: cluster_sizeX(cluster_sizeX_), cluster_sizeY(cluster_sizeY_),
dt(dt_) {
m_data = new std::byte[cluster_sizeX * cluster_sizeY * dt.bytes()]{};
}
Cluster() : Cluster(3, 3) {}
Cluster(const Cluster &other)
: Cluster(other.cluster_sizeX, other.cluster_sizeY, other.dt) {
if (this == &other)
return;
x = other.x;
y = other.y;
memcpy(m_data, other.m_data, other.bytes());
}
Cluster &operator=(const Cluster &other) {
if (this == &other)
return *this;
this->~Cluster();
new (this) Cluster(other);
return *this;
}
Cluster(Cluster &&other) noexcept
: cluster_sizeX(other.cluster_sizeX),
cluster_sizeY(other.cluster_sizeY), x(other.x), y(other.y),
dt(other.dt), m_data(other.m_data) {
other.m_data = nullptr;
other.dt = Dtype(Dtype::TypeIndex::ERROR);
}
~Cluster() { delete[] m_data; }
template <typename T> T get(int idx) {
(sizeof(T) == dt.bytes())
? 0
: throw std::invalid_argument("[ERROR] Type size mismatch");
return *reinterpret_cast<T *>(m_data + idx * dt.bytes());
}
template <typename T> auto set(int idx, T val) {
(sizeof(T) == dt.bytes())
? 0
: throw std::invalid_argument("[ERROR] Type size mismatch");
return memcpy(m_data + idx * dt.bytes(), &val, (size_t)dt.bytes());
}
// auto x() const { return x; }
// auto y() const { return y; }
// auto x(int16_t x_) { return x = x_; }
// auto y(int16_t y_) { return y = y_; }
template <typename T> std::string to_string() const {
(sizeof(T) == dt.bytes())
? 0
: throw std::invalid_argument("[ERROR] Type size mismatch");
std::string s = "x: " + std::to_string(x) + " y: " + std::to_string(y) +
"\nm_data: [";
for (int i = 0; i < cluster_sizeX * cluster_sizeY; i++) {
s += std::to_string(
*reinterpret_cast<T *>(m_data + i * dt.bytes())) +
" ";
}
s += "]";
return s;
}
/**
* @brief size of the cluster in bytes when saved to a file
*/
size_t size() const { return cluster_sizeX * cluster_sizeY; }
size_t bytes() const { return cluster_sizeX * cluster_sizeY * dt.bytes(); }
auto begin() const { return m_data; }
auto end() const {
return m_data + cluster_sizeX * cluster_sizeY * dt.bytes();
}
std::byte *data() { return m_data; }
};
/**
* @brief header contained in parts of frames
*/
struct DetectorHeader {
uint64_t frameNumber;
uint32_t expLength;
uint32_t packetNumber;
uint64_t bunchId;
uint64_t timestamp;
uint16_t modId;
uint16_t row;
uint16_t column;
uint16_t reserved;
uint32_t debug;
uint16_t roundRNumber;
uint8_t detType;
uint8_t version;
std::array<uint8_t, 64> packetMask;
std::string to_string() {
std::string packetMaskStr = "[";
for (auto &i : packetMask) {
packetMaskStr += std::to_string(i) + ", ";
}
packetMaskStr += "]";
return "frameNumber: " + std::to_string(frameNumber) + "\n" +
"expLength: " + std::to_string(expLength) + "\n" +
"packetNumber: " + std::to_string(packetNumber) + "\n" +
"bunchId: " + std::to_string(bunchId) + "\n" +
"timestamp: " + std::to_string(timestamp) + "\n" +
"modId: " + std::to_string(modId) + "\n" +
"row: " + std::to_string(row) + "\n" +
"column: " + std::to_string(column) + "\n" +
"reserved: " + std::to_string(reserved) + "\n" +
"debug: " + std::to_string(debug) + "\n" +
"roundRNumber: " + std::to_string(roundRNumber) + "\n" +
"detType: " + std::to_string(detType) + "\n" +
"version: " + std::to_string(version) + "\n" +
"packetMask: " + packetMaskStr + "\n";
}
};
template <typename T> struct t_xy {
T row;
T col;
bool operator==(const t_xy &other) const {
return row == other.row && col == other.col;
}
bool operator!=(const t_xy &other) const { return !(*this == other); }
std::string to_string() const {
return "{ x: " + std::to_string(row) + " y: " + std::to_string(col) +
" }";
}
};
using xy = t_xy<uint32_t>;
using dynamic_shape = std::vector<int64_t>;
//TODO! Can we uniform enums between the libraries?
enum class DetectorType {
Jungfrau,
Eiger,
Mythen3,
Moench,
Moench03,
Moench03_old,
ChipTestBoard,
Unknown
};
enum class TimingMode { Auto, Trigger };
enum class FrameDiscardPolicy { NoDiscard, Discard, DiscardPartial };
template <class T> T StringTo(const std::string &arg) { return T(arg); }
template <class T> std::string ToString(T arg) { return T(arg); }
template <> DetectorType StringTo(const std::string & /*name*/);
template <> std::string ToString(DetectorType arg);
template <> TimingMode StringTo(const std::string & /*mode*/);
template <> FrameDiscardPolicy StringTo(const std::string & /*mode*/);
using DataTypeVariants = std::variant<uint16_t, uint32_t>;
} // namespace aare

14
pyproject.toml Normal file
View File

@ -0,0 +1,14 @@
[build-system]
requires = ["scikit-build-core>=0.10", "pybind11", "numpy"]
build-backend = "scikit_build_core.build"
[project]
name = "aare"
version = "2024.11.07.dev0"
[tool.scikit-build]
cmake.verbose = true
[tool.scikit-build.cmake.define]
AARE_PYTHON_BINDINGS = "ON"
AARE_SYSTEM_LIBRARIES = "ON"

50
python/CMakeLists.txt Normal file
View File

@ -0,0 +1,50 @@
find_package (Python 3.10 COMPONENTS Interpreter Development)
# Download or find pybind11 depending on configuration
if(AARE_FETCH_PYBIND11)
FetchContent_Declare(
pybind11
GIT_REPOSITORY https://github.com/pybind/pybind11
GIT_TAG v2.13.0
)
FetchContent_MakeAvailable(pybind11)
else()
find_package(pybind11 2.13 REQUIRED)
endif()
# Add the compiled python extension
pybind11_add_module(
_aare # name of the module
src/module.cpp # source file
)
set_target_properties(_aare PROPERTIES
LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
)
target_link_libraries(_aare PRIVATE aare_core aare_compiler_flags)
# List of python files to be copied to the build directory
set( PYTHON_FILES
aare/__init__.py
aare/CtbRawFile.py
aare/transform.py
aare/ScanParameters.py
)
# Copy the python files to the build directory
foreach(FILE ${PYTHON_FILES})
configure_file(${FILE} ${CMAKE_BINARY_DIR}/${FILE} )
endforeach(FILE ${PYTHON_FILES})
set_target_properties(_aare PROPERTIES
LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/aare
)
# Copy the examples/scripts to the build directory
configure_file(examples/play.py ${CMAKE_BINARY_DIR}/play.py)
install(TARGETS _aare DESTINATION aare)

171
python/aare/CtbRawFile.py Normal file
View File

@ -0,0 +1,171 @@
from . import _aare
import numpy as np
from .ScanParameters import ScanParameters
class CtbRawFile(_aare.CtbRawFile):
"""File reader for the CTB raw file format.
Args:
fname (pathlib.Path | str): Path to the file to be read.
transform (function): Function to apply to the data after reading it.
The function should take a numpy array of type uint8 and return one
or several numpy arrays.
"""
def __init__(self, fname, transform = None):
super().__init__(fname)
self.transform = transform
def read_frame(self, frame_index: int | None = None ) -> tuple:
"""Read one frame from the file and then advance the file pointer.
.. note::
Uses the position of the file pointer :py:meth:`~CtbRawFile.tell` to determine
which frame to read unless frame_index is specified.
Args:
frame_index (int): If not None, seek to this frame before reading.
Returns:
tuple: header, data
Raises:
RuntimeError: If the file is at the end.
"""
if frame_index is not None:
self.seek(frame_index)
header, data = super().read_frame()
if header.shape == (1,):
header = header[0]
if self.transform:
res = self.transform(data)
if isinstance(res, tuple):
return header, *res
else:
return header, res
else:
return header, data
def read_n(self, n_frames:int) -> tuple:
"""Read several frames from the file.
.. note::
Uses the position of the file pointer :py:meth:`~CtbRawFile.tell` to determine
where to start reading from.
Args:
n_frames (int): Number of frames to read.
Returns:
tuple: header, data
Raises:
RuntimeError: If EOF is reached.
"""
# Do the first read to figure out what we have
tmp_header, tmp_data = self.read_frame()
# Allocate arrays for
header = np.zeros(n_frames, dtype = tmp_header.dtype)
data = np.zeros((n_frames, *tmp_data.shape), dtype = tmp_data.dtype)
# Copy the first frame
header[0] = tmp_header
data[0] = tmp_data
# Do the rest of the reading
for i in range(1, n_frames):
header[i], data[i] = self.read_frame()
return header, data
def read(self) -> tuple:
"""Read the entire file.
Returns:
tuple: header, data
"""
return self.read_n(self.frames_in_file)
def seek(self, frame_index:int) -> None:
"""Seek to a specific frame in the file.
Args:
frame_index (int): Frame position in file to seek to.
"""
super().seek(frame_index)
def tell() -> int:
"""Return the current frame position in the file.
Returns:
int: Frame position in file.
"""
return super().tell()
@property
def scan_parameters(self):
"""Return the scan parameters.
Returns:
ScanParameters: Scan parameters.
"""
return ScanParameters(self.master.scan_parameters)
@property
def master(self):
"""Return the master file.
Returns:
RawMasterFile: Master file.
"""
return super().master()
@property
def image_size_in_bytes(self) -> int:
"""Return the size of the image in bytes.
Returns:
int: Size of image in bytes.
"""
return super().image_size_in_bytes
def __len__(self) -> int:
"""Return the number of frames in the file.
Returns:
int: Number of frames in file.
"""
return super().frames_in_file
@property
def frames_in_file(self) -> int:
"""Return the number of frames in the file.
Returns:
int: Number of frames in file.
"""
return super().frames_in_file
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
pass
def __iter__(self):
return self
def __next__(self):
try:
return self.read_frame()
except RuntimeError:
# TODO! find a good way to check that we actually have the right exception
raise StopIteration

16
python/aare/ScanParameters.py Normal file
View File

@ -0,0 +1,16 @@
from . import _aare
class ScanParameters(_aare.ScanParameters):
def __init__(self, s):
super().__init__(s)
def __iter__(self):
return [getattr(self, a) for a in ['start', 'stop', 'step']].__iter__()
def __str__(self):
return f'ScanParameters({self.dac}: {self.start}, {self.stop}, {self.step})'
def __repr__(self):
return self.__str__()

7
python/aare/__init__.py Normal file
View File

@ -0,0 +1,7 @@
# Make the compiled classes that live in _aare available from aare.
from . import _aare
from ._aare import VarClusterFinder, File, RawMasterFile
from ._aare import Pedestal, ClusterFinder
from .CtbRawFile import CtbRawFile
from .ScanParameters import ScanParameters

28
python/aare/transform.py Normal file
View File

@ -0,0 +1,28 @@
import numpy as np
from . import _aare
class Moench05Transform:
#Could be moved to C++ without changing the interface
def __init__(self):
self.pixel_map = _aare.GenerateMoench05PixelMap()
def __call__(self, data):
return np.take(data.view(np.uint16), self.pixel_map)
class Matterhorn02Transform:
def __init__(self):
self.pixel_map = _aare.GenerateMH02FourCounterPixelMap()
def __call__(self, data):
counters = int(data.size / 48**2 / 2)
if counters == 1:
return np.take(data.view(np.uint16), self.pixel_map[0])
else:
return np.take(data.view(np.uint16), self.pixel_map[0:counters])
#on import generate the pixel maps to avoid doing it every time
moench05 = Moench05Transform()
matterhorn02 = Matterhorn02Transform()

136
python/examples/play.py Normal file
View File

@ -0,0 +1,136 @@
import matplotlib.pyplot as plt
import numpy as np
plt.ion()
import aare
from aare import CtbRawFile
print('aare imported')
from aare import transform
print('transform imported')
from pathlib import Path
import json
def decode(frames, rawdata):
# rawdata = np.fromfile(f, dtype = np.uint16)
counters = int((np.shape(rawdata)[0]/frames-56)/(48*48))
print('Counters:', counters)
rawdata = rawdata.reshape(frames,-1)[:,56:]
rawdata = rawdata.reshape(frames,576*counters,4) #Data come in "blocks" of 4 pixels/receiver
tr1 = rawdata[:,0:576*counters:2] #Transceiver1
tr1=tr1.reshape((frames,48*counters,24))
tr2 = rawdata[:,1:576*counters:2] #Transceiver2
tr2=tr2.reshape((frames,48*counters,24))
data = np.append(tr1,tr2,axis=2)
return data
def get_Mh02_frames(fname):
# this function gives you the data from a file that is not a scan
# it returns a (frames,48*counters,48)
jsonf = open(fname)
jsonpar = json.load(jsonf)
jsonf.close()
frames=jsonpar["Frames in File"]
print('Frames:', frames)
rawf = fname.replace('master','d0_f0')
rawf = rawf.replace('.json','.raw')
with open(rawf, 'rb') as f:
rawdata = np.fromfile(f, dtype = np.uint16)
data = decode(frames, rawdata)
print('Data:', np.shape(data))
return data
#target format
# [frame, counter, row, col]
# plt.imshow(data[0,0])
# p = Path('/Users/erik/data/aare_test_data/jungfrau/jungfrau_single_master_0.json')
# f = aare.File(p)
# frame = f.read_frame()
# fig, ax = plt.subplots()
# im = ax.imshow(frame, cmap='viridis')
# fpath = Path('/Users/erik/data/Moench03old/test_034_irradiated_noise_g4_hg_exptime_2000us_master_0.json')
# # fpath = Path('/Users/erik/data/Moench05/moench05_multifile_master_0.json')
# # f = aare.CtbRawFile(fpath, transform = transform.moench05)
# # with CtbRawFile(fpath, transform = transform.moench05) as f:
# # for header, image in f:
# # print(f'Frame number: {header["frameNumber"]}')
# # m = aare.RawMasterFile(fpath)
# f = aare.File(fpath)
# cf = aare.ClusterFinder((400,400),(3,3))
# for i in range(100):
# cf.push_pedestal_frame(f.read_frame())
# f.seek(0)
# pd = f.read_n(100).mean(axis=0)
# clusters = cf.find_clusters_without_threshold(f.read_frame())
base = Path('/Users/erik/data/matterhorn/raw')
fpath = Path(base / 'scan_15keV_vrf700_vrsh700_th0_master_0.json')
f = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
f.seek(100)
header1, image1 = f.read_frame()
# fpath = Path(base / 'scan_all15keV_vrf500_vrsh700_th0_master_0.json')
# f = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
# f.seek(100)
# header4, image4 = f.read_frame()
# n_counters = image.shape[1] / 48**2 / 2
# for i in range(100):
# header, image = f.read_frame()
# print(header['frameNumber'])
#Data come in "blocks" of 4 pixels/receiver
# data = get_Mh02_frames(fpath.as_posix())
# rawi = np.zeros(48*48*4+56, dtype = np.uint16)
# for i,v in enumerate(rawi[56:]):
# rawi[i+56] = i
# raw = image.view(np.uint16)
# pixel_map = decode(1, rawi)
# # img = np.take(raw, pixel_map)
# pm = np.zeros((4, 48,48), dtype = np.int64)
# for counter in range(4):
# for row in range(48):
# for col in range(48):
# pm[counter, row, col] = row*48 + col+counter*48*48
# f2 = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
# header, data = f2.read()
# plt.plot(data[:,0,20,20])

52
python/src/cluster.hpp Normal file
View File

@ -0,0 +1,52 @@
#include "aare/ClusterFinder.hpp"
#include "aare/NDView.hpp"
#include "aare/Pedestal.hpp"
#include "np_helper.hpp"
#include <cstdint>
#include <filesystem>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
namespace py = pybind11;
void define_cluster_finder_bindings(py::module &m) {
py::class_<ClusterFinder<uint16_t, double>>(m, "ClusterFinder")
.def(py::init<Shape<2>, Shape<2>>())
.def("push_pedestal_frame",
[](ClusterFinder<uint16_t, double> &self,
py::array_t<uint16_t> frame) {
auto view = make_view_2d(frame);
self.push_pedestal_frame(view);
})
.def("pedestal",
[](ClusterFinder<uint16_t, double> &self) {
auto m = new NDArray<double, 2>{};
*m = self.pedestal();
return return_image_data(m);
})
.def("find_clusters_without_threshold",
[](ClusterFinder<uint16_t, double> &self,
py::array_t<uint16_t> frame) {
auto view = make_view_2d(frame);
auto clusters = self.find_clusters_without_threshold(view);
return clusters;
});
py::class_<Cluster>(m, "Cluster", py::buffer_protocol())
.def(py::init<int, int, Dtype>())
.def("size", &Cluster::size)
.def("begin", &Cluster::begin)
.def("end", &Cluster::end)
.def_readwrite("x", &Cluster::x)
.def_readwrite("y", &Cluster::y)
.def_buffer([](Cluster &c) -> py::buffer_info {
return py::buffer_info(c.data(), c.dt.bytes(), c.dt.format_descr(),
1, {c.size()}, {c.dt.bytes()});
})
.def("__repr__", [](const Cluster &a) {
return "<Cluster: x: " + std::to_string(a.x) +
", y: " + std::to_string(a.y) + ">";
});
}

240
python/src/file.hpp Normal file
View File

@ -0,0 +1,240 @@
#include "aare/CtbRawFile.hpp"
#include "aare/RawMasterFile.hpp"
#include "aare/File.hpp"
#include "aare/Frame.hpp"
#include "aare/defs.hpp"
// #include "aare/fClusterFileV2.hpp"
#include <cstdint>
#include <filesystem>
#include <pybind11/iostream.h>
#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/stl/filesystem.h>
#include <string>
namespace py = pybind11;
using namespace ::aare;
void define_file_io_bindings(py::module &m) {
PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber,
bunchId, timestamp, modId, row, column, reserved,
debug, roundRNumber, detType, version, packetMask);
py::class_<CtbRawFile>(m, "CtbRawFile")
.def(py::init<const std::filesystem::path &>())
.def("read_frame",
[](CtbRawFile &self) {
size_t image_size = self.image_size_in_bytes();
py::array image;
std::vector<ssize_t> shape;
shape.reserve(2);
shape.push_back(1);
shape.push_back(image_size);
py::array_t<DetectorHeader> header(1);
// always read bytes
image = py::array_t<uint8_t>(shape);
self.read_into(
reinterpret_cast<std::byte *>(image.mutable_data()),
header.mutable_data());
return py::make_tuple(header, image);
})
.def("seek", &CtbRawFile::seek)
.def("tell", &CtbRawFile::tell)
.def("master", &CtbRawFile::master)
.def_property_readonly("image_size_in_bytes", &CtbRawFile::image_size_in_bytes)
.def_property_readonly("frames_in_file", &CtbRawFile::frames_in_file);
py::class_<File>(m, "File")
.def(py::init([](const std::filesystem::path &fname) {
return File(fname, "r", {});
}))
.def(py::init(
[](const std::filesystem::path &fname, const std::string &mode) {
return File(fname, mode, {});
}))
.def(py::init<const std::filesystem::path &, const std::string &,
const FileConfig &>())
.def("frame_number", &File::frame_number)
.def_property_readonly("bytes_per_frame", &File::bytes_per_frame)
.def_property_readonly("pixels_per_frame", &File::pixels_per_frame)
.def("seek", &File::seek)
.def("tell", &File::tell)
.def_property_readonly("total_frames", &File::total_frames)
.def_property_readonly("rows", &File::rows)
.def_property_readonly("cols", &File::cols)
.def_property_readonly("bitdepth", &File::bitdepth)
.def_property_readonly("bytes_per_pixel", &File::bytes_per_pixel)
.def_property_readonly(
"detector_type",
[](File &self) { return ToString(self.detector_type()); })
.def("read_frame",
[](File &self) {
const uint8_t item_size = self.bytes_per_pixel();
py::array image;
std::vector<ssize_t> shape;
shape.reserve(2);
shape.push_back(self.rows());
shape.push_back(self.cols());
if (item_size == 1) {
image = py::array_t<uint8_t>(shape);
} else if (item_size == 2) {
image = py::array_t<uint16_t>(shape);
} else if (item_size == 4) {
image = py::array_t<uint32_t>(shape);
}
self.read_into(
reinterpret_cast<std::byte *>(image.mutable_data()));
return image;
})
.def("read_frame",
[](File &self, size_t frame_number) {
self.seek(frame_number);
const uint8_t item_size = self.bytes_per_pixel();
py::array image;
std::vector<ssize_t> shape;
shape.reserve(2);
shape.push_back(self.rows());
shape.push_back(self.cols());
if (item_size == 1) {
image = py::array_t<uint8_t>(shape);
} else if (item_size == 2) {
image = py::array_t<uint16_t>(shape);
} else if (item_size == 4) {
image = py::array_t<uint32_t>(shape);
}
self.read_into(
reinterpret_cast<std::byte *>(image.mutable_data()));
return image;
})
.def("read_n", [](File &self, size_t n_frames) {
const uint8_t item_size = self.bytes_per_pixel();
py::array image;
std::vector<ssize_t> shape;
shape.reserve(3);
shape.push_back(n_frames);
shape.push_back(self.rows());
shape.push_back(self.cols());
if (item_size == 1) {
image = py::array_t<uint8_t>(shape);
} else if (item_size == 2) {
image = py::array_t<uint16_t>(shape);
} else if (item_size == 4) {
image = py::array_t<uint32_t>(shape);
}
self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()),
n_frames);
return image;
});
py::class_<FileConfig>(m, "FileConfig")
.def(py::init<>())
.def_readwrite("rows", &FileConfig::rows)
.def_readwrite("cols", &FileConfig::cols)
.def_readwrite("version", &FileConfig::version)
.def_readwrite("geometry", &FileConfig::geometry)
.def_readwrite("detector_type", &FileConfig::detector_type)
.def_readwrite("max_frames_per_file", &FileConfig::max_frames_per_file)
.def_readwrite("total_frames", &FileConfig::total_frames)
.def_readwrite("dtype", &FileConfig::dtype)
.def("__eq__", &FileConfig::operator==)
.def("__ne__", &FileConfig::operator!=)
.def("__repr__", [](const FileConfig &a) {
return "<FileConfig: " + a.to_string() + ">";
});
py::class_<RawMasterFile>(m, "RawMasterFile")
.def(py::init<const std::filesystem::path &>())
.def("data_fname", &RawMasterFile::data_fname)
.def_property_readonly("version", &RawMasterFile::version)
.def_property_readonly("detector_type", &RawMasterFile::detector_type)
.def_property_readonly("timing_mode", &RawMasterFile::timing_mode)
.def_property_readonly("image_size_in_bytes",
&RawMasterFile::image_size_in_bytes)
.def_property_readonly("frames_in_file", &RawMasterFile::frames_in_file)
.def_property_readonly("pixels_y", &RawMasterFile::pixels_y)
.def_property_readonly("pixels_x", &RawMasterFile::pixels_x)
.def_property_readonly("max_frames_per_file",
&RawMasterFile::max_frames_per_file)
.def_property_readonly("bitdepth", &RawMasterFile::bitdepth)
.def_property_readonly("frame_padding", &RawMasterFile::frame_padding)
.def_property_readonly("frame_discard_policy",
&RawMasterFile::frame_discard_policy)
.def_property_readonly("total_frames_expected", &RawMasterFile::total_frames_expected)
.def_property_readonly("geometry", &RawMasterFile::geometry)
.def_property_readonly("analog_samples", &RawMasterFile::analog_samples)
.def_property_readonly("digital_samples",
&RawMasterFile::digital_samples)
.def_property_readonly("transceiver_samples", &RawMasterFile::transceiver_samples)
.def_property_readonly("number_of_rows", &RawMasterFile::number_of_rows)
.def_property_readonly("quad", &RawMasterFile::quad)
.def_property_readonly("scan_parameters", &RawMasterFile::scan_parameters);
py::class_<ScanParameters>(m, "ScanParameters")
.def(py::init<const std::string &>())
.def(py::init<const ScanParameters&>())
.def_property_readonly("enabled", &ScanParameters::enabled)
.def_property_readonly("dac", &ScanParameters::dac)
.def_property_readonly("start", &ScanParameters::start)
.def_property_readonly("stop", &ScanParameters::stop)
.def_property_readonly("step", &ScanParameters::step);
// py::class_<ClusterHeader>(m, "ClusterHeader")
// .def(py::init<>())
// .def_readwrite("frame_number", &ClusterHeader::frame_number)
// .def_readwrite("n_clusters", &ClusterHeader::n_clusters)
// .def("__repr__", [](const ClusterHeader &a) { return "<ClusterHeader:
// " + a.to_string() + ">"; });
// py::class_<ClusterV2_>(m, "ClusterV2_")
// .def(py::init<>())
// .def_readwrite("x", &ClusterV2_::x)
// .def_readwrite("y", &ClusterV2_::y)
// .def_readwrite("data", &ClusterV2_::data)
// .def("__repr__", [](const ClusterV2_ &a) { return "<ClusterV2_: " +
// a.to_string(false) + ">"; });
// py::class_<ClusterV2>(m, "ClusterV2")
// .def(py::init<>())
// .def_readwrite("cluster", &ClusterV2::cluster)
// .def_readwrite("frame_number", &ClusterV2::frame_number)
// .def("__repr__", [](const ClusterV2 &a) { return "<ClusterV2: " +
// a.to_string() + ">"; });
// py::class_<ClusterFileV2>(m, "ClusterFileV2")
// .def(py::init<const std::filesystem::path &, const std::string &>())
// .def("read", py::overload_cast<>(&ClusterFileV2::read))
// .def("read", py::overload_cast<int>(&ClusterFileV2::read))
// .def("frame_number", &ClusterFileV2::frame_number)
// .def("write", py::overload_cast<std::vector<ClusterV2> const
// &>(&ClusterFileV2::write))
// .def("close", &ClusterFileV2::close);
// m.def("to_clustV2", [](std::vector<Cluster> &clusters, const int
// frame_number) {
// std::vector<ClusterV2> clusters_;
// for (auto &c : clusters) {
// ClusterV2 cluster;
// cluster.cluster.x = c.x;
// cluster.cluster.y = c.y;
// int i=0;
// for(auto &d : cluster.cluster.data) {
// d=c.get<double>(i++);
// }
// cluster.frame_number = frame_number;
// clusters_.push_back(cluster);
// }
// return clusters_;
// });
}

21
python/src/module.cpp Normal file
View File

@ -0,0 +1,21 @@
//Files with bindings to the different classes
#include "file.hpp"
#include "var_cluster.hpp"
#include "pixel_map.hpp"
#include "pedestal.hpp"
#include "cluster.hpp"
//Pybind stuff
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
namespace py = pybind11;
PYBIND11_MODULE(_aare, m) {
define_file_io_bindings(m);
define_var_cluster_finder_bindings(m);
define_pixel_map_bindings(m);
define_pedestal_bindings<double>(m, "Pedestal");
define_pedestal_bindings<float>(m, "Pedestal_float32");
define_cluster_finder_bindings(m);
}

116
python/src/np_helper.hpp Normal file
View File

@ -0,0 +1,116 @@
#pragma once
#include <iostream>
#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include "aare/Frame.hpp"
#include "aare/NDArray.hpp"
#include "aare/NDView.hpp"
namespace py = pybind11;
// Pass image data back to python as a numpy array
template <typename T, int64_t Ndim>
py::array return_image_data(aare::NDArray<T, Ndim> *image) {
py::capsule free_when_done(image, [](void *f) {
aare::NDArray<T, Ndim> *foo =
reinterpret_cast<aare::NDArray<T, Ndim> *>(f);
delete foo;
});
return py::array_t<T>(
image->shape(), // shape
image->byte_strides(), // C-style contiguous strides for double
image->data(), // the data pointer
free_when_done); // numpy array references this parent
}
template <typename T> py::array return_vector(std::vector<T> *vec) {
py::capsule free_when_done(vec, [](void *f) {
std::vector<T> *foo = reinterpret_cast<std::vector<T> *>(f);
delete foo;
});
return py::array_t<T>({vec->size()}, // shape
{sizeof(T)}, // C-style contiguous strides for double
vec->data(), // the data pointer
free_when_done); // numpy array references this parent
}
// template <typename Reader> py::array do_read(Reader &r, size_t n_frames) {
// py::array image;
// if (n_frames == 0)
// n_frames = r.total_frames();
// std::array<ssize_t, 3> shape{static_cast<ssize_t>(n_frames), r.rows(),
// r.cols()};
// const uint8_t item_size = r.bytes_per_pixel();
// if (item_size == 1) {
// image = py::array_t<uint8_t, py::array::c_style | py::array::forcecast>(
// shape);
// } else if (item_size == 2) {
// image =
// py::array_t<uint16_t, py::array::c_style | py::array::forcecast>(
// shape);
// } else if (item_size == 4) {
// image =
// py::array_t<uint32_t, py::array::c_style | py::array::forcecast>(
// shape);
// }
// r.read_into(reinterpret_cast<std::byte *>(image.mutable_data()), n_frames);
// return image;
// }
// py::array return_frame(pl::Frame *ptr) {
// py::capsule free_when_done(ptr, [](void *f) {
// pl::Frame *foo = reinterpret_cast<pl::Frame *>(f);
// delete foo;
// });
// const uint8_t item_size = ptr->bytes_per_pixel();
// std::vector<ssize_t> shape;
// for (auto val : ptr->shape())
// if (val > 1)
// shape.push_back(val);
// std::vector<ssize_t> strides;
// if (shape.size() == 1)
// strides.push_back(item_size);
// else if (shape.size() == 2) {
// strides.push_back(item_size * shape[1]);
// strides.push_back(item_size);
// }
// if (item_size == 1)
// return py::array_t<uint8_t>(
// shape, strides,
// reinterpret_cast<uint8_t *>(ptr->data()), free_when_done);
// else if (item_size == 2)
// return py::array_t<uint16_t>(shape, strides,
// reinterpret_cast<uint16_t *>(ptr->data()),
// free_when_done);
// else if (item_size == 4)
// return py::array_t<uint32_t>(shape, strides,
// reinterpret_cast<uint32_t *>(ptr->data()),
// free_when_done);
// return {};
// }
// todo rewrite generic
template <class T, int Flags> auto get_shape_3d(py::array_t<T, Flags> arr) {
return aare::Shape<3>{arr.shape(0), arr.shape(1), arr.shape(2)};
}
template <class T, int Flags> auto make_view_3d(py::array_t<T, Flags> arr) {
return aare::NDView<T, 3>(arr.mutable_data(), get_shape_3d<T, Flags>(arr));
}
template <class T, int Flags> auto get_shape_2d(py::array_t<T, Flags> arr) {
return aare::Shape<2>{arr.shape(0), arr.shape(1)};
}
template <class T, int Flags> auto make_view_2d(py::array_t<T, Flags> arr) {
return aare::NDView<T, 2>(arr.mutable_data(), get_shape_2d<T, Flags>(arr));
}

47
python/src/pedestal.hpp Normal file
View File

@ -0,0 +1,47 @@
#include "aare/Pedestal.hpp"
#include "np_helper.hpp"
#include <cstdint>
#include <filesystem>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
namespace py = pybind11;
template <typename SUM_TYPE> void define_pedestal_bindings(py::module &m, const std::string &name) {
py::class_<Pedestal<SUM_TYPE>>(m, name.c_str())
.def(py::init<int, int, int>())
.def(py::init<int, int>())
.def("mean",
[](Pedestal<SUM_TYPE> &self) {
auto m = new NDArray<SUM_TYPE, 2>{};
*m = self.mean();
return return_image_data(m);
})
.def("variance", [](Pedestal<SUM_TYPE> &self) {
auto m = new NDArray<SUM_TYPE, 2>{};
*m = self.variance();
return return_image_data(m);
})
.def("std", [](Pedestal<SUM_TYPE> &self) {
auto m = new NDArray<SUM_TYPE, 2>{};
*m = self.std();
return return_image_data(m);
})
.def("clear", py::overload_cast<>(&Pedestal<SUM_TYPE>::clear))
.def_property_readonly("rows", &Pedestal<SUM_TYPE>::rows)
.def_property_readonly("cols", &Pedestal<SUM_TYPE>::cols)
.def_property_readonly("n_samples", &Pedestal<SUM_TYPE>::n_samples)
.def_property_readonly("sum", &Pedestal<SUM_TYPE>::get_sum)
.def_property_readonly("sum2", &Pedestal<SUM_TYPE>::get_sum2)
.def("clone",
[&](Pedestal<SUM_TYPE> &pedestal) {
return Pedestal<SUM_TYPE>(pedestal);
})
//TODO! add push for other data types
.def("push", [](Pedestal<SUM_TYPE> &pedestal, py::array_t<uint16_t> &f) {
auto v = make_view_2d(f);
pedestal.push(v);
});
}

33
python/src/pixel_map.hpp Normal file
View File

@ -0,0 +1,33 @@
#include "aare/PixelMap.hpp"
#include "np_helper.hpp"
#include <cstdint>
#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
namespace py = pybind11;
using namespace::aare;
void define_pixel_map_bindings(py::module &m) {
m.def("GenerateMoench03PixelMap", []() {
auto ptr = new NDArray<ssize_t,2>(GenerateMoench03PixelMap());
return return_image_data(ptr);
})
.def("GenerateMoench05PixelMap", []() {
auto ptr = new NDArray<ssize_t,2>(GenerateMoench05PixelMap());
return return_image_data(ptr);
})
.def("GenerateMH02SingleCounterPixelMap", []() {
auto ptr = new NDArray<ssize_t,2>(GenerateMH02SingleCounterPixelMap());
return return_image_data(ptr);
})
.def("GenerateMH02FourCounterPixelMap", []() {
auto ptr = new NDArray<ssize_t,3>(GenerateMH02FourCounterPixelMap());
return return_image_data(ptr);
});
}

46
python/src/var_cluster.hpp Normal file
View File

@ -0,0 +1,46 @@
#include "aare/VarClusterFinder.hpp"
#include "np_helper.hpp"
// #include "aare/defs.hpp"
// #include "aare/fClusterFileV2.hpp"
#include <cstdint>
// #include <filesystem>
#include <pybind11/numpy.h>
// #include <pybind11/iostream.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
// #include <pybind11/stl/filesystem.h>
// #include <string>
namespace py = pybind11;
using namespace::aare;
void define_var_cluster_finder_bindings(py::module &m) {
PYBIND11_NUMPY_DTYPE(VarClusterFinder<double>::Hit, size, row, col,
reserved, energy, max);
py::class_<VarClusterFinder<double>>(m, "VarClusterFinder")
.def(py::init<Shape<2>, double>())
.def("labeled",
[](VarClusterFinder<double> &self) {
auto ptr = new NDArray<int, 2>(self.labeled());
return return_image_data(ptr);
})
.def("find_clusters",
[](VarClusterFinder<double> &self,
py::array_t<double, py::array::c_style | py::array::forcecast>
img) {
auto view = make_view_2d(img);
self.find_clusters(view);
})
.def("steal_hits",
[](VarClusterFinder<double> &self) {
auto ptr = new std::vector<VarClusterFinder<double>::Hit>(
self.steal_hits());
return return_vector(ptr);
})
.def("total_clusters", &VarClusterFinder<double>::total_clusters);
}

71
src/ClusterFinder.test.cpp Normal file
View File

@ -0,0 +1,71 @@
#include "aare/ClusterFinder.hpp"
#include "aare/Pedestal.hpp"
#include <catch2/matchers/catch_matchers_floating_point.hpp>
#include <catch2/catch_test_macros.hpp>
#include <chrono>
#include <random>
using namespace aare;
//TODO! Find a way to test the cluster finder
// class ClusterFinderUnitTest : public ClusterFinder {
// public:
// ClusterFinderUnitTest(int cluster_sizeX, int cluster_sizeY, double nSigma = 5.0, double threshold = 0.0)
// : ClusterFinder(cluster_sizeX, cluster_sizeY, nSigma, threshold) {}
// double get_c2() { return c2; }
// double get_c3() { return c3; }
// auto get_threshold() { return m_threshold; }
// auto get_nSigma() { return m_nSigma; }
// auto get_cluster_sizeX() { return m_cluster_sizeX; }
// auto get_cluster_sizeY() { return m_cluster_sizeY; }
// };
// TEST_CASE("test ClusterFinder constructor") {
// ClusterFinderUnitTest cf(55, 100);
// REQUIRE(cf.get_cluster_sizeX() == 55);
// REQUIRE(cf.get_cluster_sizeY() == 100);
// REQUIRE(cf.get_threshold() == 0.0);
// REQUIRE(cf.get_nSigma() == 5.0);
// double c2 = sqrt((100 + 1) / 2 * (55 + 1) / 2);
// double c3 = sqrt(55 * 100);
// // REQUIRE(compare_floats<double>(cf.get_c2(), c2));
// // REQUIRE(compare_floats<double>(cf.get_c3(), c3));
// REQUIRE_THAT(cf.get_c2(), Catch::Matchers::WithinRel(c2, 1e-9));
// REQUIRE_THAT(cf.get_c3(), Catch::Matchers::WithinRel(c3, 1e-9));
// }
TEST_CASE("Construct a cluster finder"){
ClusterFinder clusterFinder({400,400}, {3,3});
// REQUIRE(clusterFinder.get_cluster_sizeX() == 3);
// REQUIRE(clusterFinder.get_cluster_sizeY() == 3);
// REQUIRE(clusterFinder.get_threshold() == 1);
// REQUIRE(clusterFinder.get_nSigma() == 1);
}
// TEST_CASE("test cluster finder") {
// aare::Pedestal pedestal(10, 10, 5);
// NDArray<double, 2> frame({10, 10});
// frame = 0;
// ClusterFinder clusterFinder(3, 3, 1, 1); // 3x3 cluster, 1 nSigma, 1 threshold
// auto clusters = clusterFinder.find_clusters_without_threshold(frame.span(), pedestal);
// REQUIRE(clusters.size() == 0);
// frame(5, 5) = 10;
// clusters = clusterFinder.find_clusters_without_threshold(frame.span(), pedestal);
// REQUIRE(clusters.size() == 1);
// REQUIRE(clusters[0].x == 5);
// REQUIRE(clusters[0].y == 5);
// for (int i = 0; i < 3; i++) {
// for (int j = 0; j < 3; j++) {
// if (i == 1 && j == 1)
// REQUIRE(clusters[0].get<double>(i * 3 + j) == 10);
// else
// REQUIRE(clusters[0].get<double>(i * 3 + j) == 0);
// }
// }
// }

74
src/CtbRawFile.cpp Normal file
View File

@ -0,0 +1,74 @@
#include "aare/CtbRawFile.hpp"
#include <fmt/format.h>
namespace aare {
CtbRawFile::CtbRawFile(const std::filesystem::path &fname) : m_master(fname) {
if (m_master.detector_type() != DetectorType::ChipTestBoard) {
throw std::runtime_error(LOCATION + "Not a Ctb file");
}
find_subfiles();
// open the first subfile
m_file.open(m_master.data_fname(0, 0), std::ios::binary);
}
void CtbRawFile::read_into(std::byte *image_buf, DetectorHeader* header) {
if(m_current_frame >= m_master.frames_in_file()){
throw std::runtime_error(LOCATION + "End of file reached");
}
if(m_current_frame != 0 && m_current_frame % m_master.max_frames_per_file() == 0){
open_data_file(m_current_subfile+1);
}
if(header){
m_file.read(reinterpret_cast<char *>(header), sizeof(DetectorHeader));
}else{
m_file.seekg(sizeof(DetectorHeader), std::ios::cur);
}
m_file.read(reinterpret_cast<char *>(image_buf), m_master.image_size_in_bytes());
m_current_frame++;
}
void CtbRawFile::seek(size_t frame_number) {
if (auto index = sub_file_index(frame_number); index != m_current_subfile) {
open_data_file(index);
}
size_t frame_number_in_file = frame_number % m_master.max_frames_per_file();
m_file.seekg((sizeof(DetectorHeader)+m_master.image_size_in_bytes()) * frame_number_in_file);
m_current_frame = frame_number;
}
size_t CtbRawFile::tell() const { return m_current_frame; }
size_t CtbRawFile::image_size_in_bytes() const { return m_master.image_size_in_bytes(); }
size_t CtbRawFile::frames_in_file() const { return m_master.frames_in_file(); }
RawMasterFile CtbRawFile::master() const { return m_master; }
void CtbRawFile::find_subfiles() {
// we can semi safely assume that there is only one module for CTB
while (std::filesystem::exists(m_master.data_fname(0, m_num_subfiles)))
m_num_subfiles++;
fmt::print("Found {} subfiles\n", m_num_subfiles);
}
void CtbRawFile::open_data_file(size_t subfile_index) {
if (subfile_index >= m_num_subfiles) {
throw std::runtime_error(LOCATION + "Subfile index out of range");
}
m_current_subfile = subfile_index;
m_file = std::ifstream(m_master.data_fname(0, subfile_index), std::ios::binary); // only one module for CTB
if (!m_file.is_open()) {
throw std::runtime_error(LOCATION + "Could not open data file");
}
}
} // namespace aare

191
src/Dtype.cpp Normal file
View File

@ -0,0 +1,191 @@
#include "aare/Dtype.hpp"
#include "aare/defs.hpp"
#include <fmt/core.h>
namespace aare {
/**
* @brief Construct a DType object from a type_info object
* @param t type_info object
* @throw runtime_error if the type is not supported
* @note supported types are: int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t, float, double
* @note the type_info object is obtained using typeid (e.g. typeid(int))
*/
Dtype::Dtype(const std::type_info &t) {
if (t == typeid(int8_t))
m_type = TypeIndex::INT8;
else if (t == typeid(uint8_t))
m_type = TypeIndex::UINT8;
else if (t == typeid(int16_t))
m_type = TypeIndex::INT16;
else if (t == typeid(uint16_t))
m_type = TypeIndex::UINT16;
else if (t == typeid(int32_t))
m_type = TypeIndex::INT32;
else if (t == typeid(uint32_t))
m_type = TypeIndex::UINT32;
else if (t == typeid(int64_t)) // NOLINT
m_type = TypeIndex::INT64;
else if (t == typeid(uint64_t))
m_type = TypeIndex::UINT64;
else if (t == typeid(float))
m_type = TypeIndex::FLOAT;
else if (t == typeid(double))
m_type = TypeIndex::DOUBLE;
else
throw std::runtime_error("Could not construct data type. Type not supported.");
}
/**
* @brief Get the bitdepth of the data type
* @return bitdepth
*/
uint8_t Dtype::bitdepth() const {
switch (m_type) {
case TypeIndex::INT8:
case TypeIndex::UINT8:
return 8;
case TypeIndex::INT16:
case TypeIndex::UINT16:
return 16;
case TypeIndex::INT32:
case TypeIndex::UINT32:
return 32;
case TypeIndex::INT64:
case TypeIndex::UINT64:
return 64;
case TypeIndex::FLOAT:
return 32;
case TypeIndex::DOUBLE:
return 64;
case TypeIndex::NONE:
return 0;
default:
throw std::runtime_error(LOCATION + "Could not get bitdepth. Type not supported.");
}
}
/**
* @brief Get the number of bytes of the data type
*/
size_t Dtype::bytes() const { return bitdepth() / 8; }
/**
* @brief Construct a DType object from a TypeIndex
* @param ti TypeIndex
*
*/
Dtype::Dtype(Dtype::TypeIndex ti) : m_type(ti) {}
/**
* @brief Construct a DType object from a string
* @param sv string_view
* @throw runtime_error if the type is not supported
* @note example strings: "<i4", "u8", "f4"
* @note the endianess is checked and only native endianess is supported
*/
Dtype::Dtype(std::string_view sv) {
// Check if the file is using our native endianess
if (auto pos = sv.find_first_of("<>"); pos != std::string_view::npos) {
const auto endianess = [](const char c) {
if (c == '<')
return endian::little;
return endian::big;
}(sv[pos]);
if (endianess != endian::native) {
throw std::runtime_error("Non native endianess not supported");
}
}
// we are done with the endianess so we can remove the prefix
sv.remove_prefix(std::min(sv.find_first_not_of("<>"), sv.size()));
if (sv == "i1")
m_type = TypeIndex::INT8;
else if (sv == "u1")
m_type = TypeIndex::UINT8;
else if (sv == "i2")
m_type = TypeIndex::INT16;
else if (sv == "u2")
m_type = TypeIndex::UINT16;
else if (sv == "i4")
m_type = TypeIndex::INT32;
else if (sv == "u4")
m_type = TypeIndex::UINT32;
else if (sv == "i8")
m_type = TypeIndex::INT64;
else if (sv == "u8")
m_type = TypeIndex::UINT64;
else if (sv == "f4")
m_type = TypeIndex::FLOAT;
else if (sv == "f8")
m_type = TypeIndex::DOUBLE;
else
throw std::runtime_error("Cannot construct data type from string.");
}
Dtype Dtype::from_bitdepth(uint8_t bitdepth) {
switch (bitdepth) {
case 8:
return Dtype(TypeIndex::UINT8);
case 16:
return Dtype(TypeIndex::UINT16);
case 32:
return Dtype(TypeIndex::UINT32);
case 64:
return Dtype(TypeIndex::UINT64);
default:
throw std::runtime_error("Could not construct data type from bitdepth.");
}
}
/**
* @brief Get the string representation of the data type
* @return string representation
*/
std::string Dtype::to_string() const {
char ec{};
if (endian::native == endian::little)
ec = '<';
else
ec = '>';
switch (m_type) {
case TypeIndex::INT8:
return fmt::format("{}i1", ec);
case TypeIndex::UINT8:
return fmt::format("{}u1", ec);
case TypeIndex::INT16:
return fmt::format("{}i2", ec);
case TypeIndex::UINT16:
return fmt::format("{}u2", ec);
case TypeIndex::INT32:
return fmt::format("{}i4", ec);
case TypeIndex::UINT32:
return fmt::format("{}u4", ec);
case TypeIndex::INT64:
return fmt::format("{}i8", ec);
case TypeIndex::UINT64:
return fmt::format("{}u8", ec);
case TypeIndex::FLOAT:
return "f4";
case TypeIndex::DOUBLE:
return "f8";
case TypeIndex::ERROR:
throw std::runtime_error("Could not get string representation. Type not supported.");
case TypeIndex::NONE:
throw std::runtime_error("Could not get string representation. Type not supported.");
}
return {};
}
bool Dtype::operator==(const Dtype &other) const noexcept { return m_type == other.m_type; }
bool Dtype::operator!=(const Dtype &other) const noexcept { return !(*this == other); }
bool Dtype::operator==(const std::type_info &t) const { return Dtype(t) == *this; }
bool Dtype::operator!=(const std::type_info &t) const { return Dtype(t) != *this; }
} // namespace aare

54
src/Dtype.test.cpp Normal file
View File

@ -0,0 +1,54 @@
#include "aare/Dtype.hpp"
#include <catch2/catch_test_macros.hpp>
using aare::Dtype;
using aare::endian;
TEST_CASE("Construct from typeid") {
REQUIRE(Dtype(typeid(int)) == typeid(int));
REQUIRE(Dtype(typeid(int)) != typeid(double));
}
TEST_CASE("Construct from string") {
if (endian::native == endian::little) {
REQUIRE(Dtype("<i1") == typeid(int8_t));
REQUIRE(Dtype("<u1") == typeid(uint8_t));
REQUIRE(Dtype("<i2") == typeid(int16_t));
REQUIRE(Dtype("<u2") == typeid(uint16_t));
REQUIRE(Dtype("<i4") == typeid(int));
REQUIRE(Dtype("<u4") == typeid(unsigned));
REQUIRE(Dtype("<i4") == typeid(int32_t));
// REQUIRE(Dtype("<i8") == typeid(long));
REQUIRE(Dtype("<i8") == typeid(int64_t));
REQUIRE(Dtype("<u4") == typeid(uint32_t));
REQUIRE(Dtype("<u8") == typeid(uint64_t));
REQUIRE(Dtype("f4") == typeid(float));
REQUIRE(Dtype("f8") == typeid(double));
}
if (endian::native == endian::big) {
REQUIRE(Dtype(">i1") == typeid(int8_t));
REQUIRE(Dtype(">u1") == typeid(uint8_t));
REQUIRE(Dtype(">i2") == typeid(int16_t));
REQUIRE(Dtype(">u2") == typeid(uint16_t));
REQUIRE(Dtype(">i4") == typeid(int));
REQUIRE(Dtype(">u4") == typeid(unsigned));
REQUIRE(Dtype(">i4") == typeid(int32_t));
// REQUIRE(Dtype(">i8") == typeid(long));
REQUIRE(Dtype(">i8") == typeid(int64_t));
REQUIRE(Dtype(">u4") == typeid(uint32_t));
REQUIRE(Dtype(">u8") == typeid(uint64_t));
REQUIRE(Dtype("f4") == typeid(float));
REQUIRE(Dtype("f8") == typeid(double));
}
}
TEST_CASE("Construct from string with endianess") {
// TODO! handle big endian system in test!
REQUIRE(Dtype("<i4") == typeid(int32_t));
REQUIRE_THROWS(Dtype(">i4") == typeid(int32_t));
}
TEST_CASE("Convert to string") { REQUIRE(Dtype(typeid(int)).to_string() == "<i4"); }

77
src/File.cpp Normal file
View File

@ -0,0 +1,77 @@
#include "aare/File.hpp"
#include "aare/NumpyFile.hpp"
#include "aare/RawFile.hpp"
#include <fmt/format.h>
namespace aare {
File::File(const std::filesystem::path &fname, const std::string &mode,
const FileConfig &cfg)
: file_impl(nullptr) {
if (mode != "r") {
throw std::invalid_argument("At the moment only reading is supported");
}
if ((mode == "r") && !std::filesystem::exists(fname)) {
throw std::runtime_error(
fmt::format("File does not exist: {}", fname.string()));
}
// Assuming we are pointing at a master file?
// TODO! How do we read raw files directly?
if (fname.extension() == ".raw" || fname.extension() == ".json") {
// file_impl = new RawFile(fname, mode, cfg);
file_impl = std::make_unique<RawFile>(fname, mode);
}
else if (fname.extension() == ".npy") {
// file_impl = new NumpyFile(fname, mode, cfg);
file_impl = std::make_unique<NumpyFile>(fname, mode, cfg);
} else {
throw std::runtime_error("Unsupported file type");
}
}
File::File(File &&other) noexcept{
std::swap(file_impl, other.file_impl);
}
File& File::operator=(File &&other) noexcept {
if (this != &other) {
File tmp(std::move(other));
std::swap(file_impl, tmp.file_impl);
}
return *this;
}
Frame File::read_frame() { return file_impl->read_frame(); }
Frame File::read_frame(size_t frame_index) {
return file_impl->read_frame(frame_index);
}
size_t File::total_frames() const { return file_impl->total_frames(); }
std::vector<Frame> File::read_n(size_t n_frames) {
return file_impl->read_n(n_frames);
}
void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); }
void File::read_into(std::byte *image_buf, size_t n_frames) {
file_impl->read_into(image_buf, n_frames);
}
size_t File::frame_number(size_t frame_index) {
return file_impl->frame_number(frame_index);
}
size_t File::bytes_per_frame() const { return file_impl->bytes_per_frame(); }
size_t File::pixels_per_frame() const{ return file_impl->pixels_per_frame(); }
void File::seek(size_t frame_index) { file_impl->seek(frame_index); }
size_t File::tell() const { return file_impl->tell(); }
size_t File::rows() const { return file_impl->rows(); }
size_t File::cols() const { return file_impl->cols(); }
size_t File::bitdepth() const { return file_impl->bitdepth(); }
size_t File::bytes_per_pixel() const { return file_impl->bitdepth() / 8; }
DetectorType File::detector_type() const { return file_impl->detector_type(); }
} // namespace aare

74
src/Frame.cpp Normal file
View File

@ -0,0 +1,74 @@
#include "aare/Frame.hpp"
#include <cstddef>
#include <cstring>
#include <iostream>
#include <sys/types.h>
namespace aare {
Frame::Frame(const std::byte *bytes, uint32_t rows, uint32_t cols, Dtype dtype)
: m_rows(rows), m_cols(cols), m_dtype(dtype),
m_data(new std::byte[rows * cols * m_dtype.bytes()]) {
std::memcpy(m_data, bytes, rows * cols * m_dtype.bytes());
}
Frame::Frame(uint32_t rows, uint32_t cols, Dtype dtype)
: m_rows(rows), m_cols(cols), m_dtype(dtype),
m_data(new std::byte[rows * cols * dtype.bytes()]) {
std::memset(m_data, 0, rows * cols * dtype.bytes());
}
uint32_t Frame::rows() const { return m_rows; }
uint32_t Frame::cols() const { return m_cols; }
size_t Frame::bitdepth() const { return m_dtype.bitdepth(); }
Dtype Frame::dtype() const { return m_dtype; }
uint64_t Frame::size() const { return m_rows * m_cols; }
size_t Frame::bytes() const { return m_rows * m_cols * m_dtype.bytes(); }
std::byte *Frame::data() const { return m_data; }
std::byte *Frame::pixel_ptr(uint32_t row, uint32_t col) const{
if ((row >= m_rows) || (col >= m_cols)) {
std::cerr << "Invalid row or column index" << '\n';
return nullptr;
}
return m_data + (row * m_cols + col) * (m_dtype.bytes());
}
Frame &Frame::operator=(Frame &&other) noexcept {
if (this == &other) {
return *this;
}
m_rows = other.rows();
m_cols = other.cols();
m_dtype = other.dtype();
if (m_data != nullptr) {
delete[] m_data;
}
m_data = other.m_data;
other.m_data = nullptr;
other.m_rows = other.m_cols = 0;
other.m_dtype = Dtype(Dtype::TypeIndex::ERROR);
return *this;
}
Frame::Frame(Frame &&other) noexcept
: m_rows(other.rows()), m_cols(other.cols()), m_dtype(other.dtype()),
m_data(other.m_data) {
other.m_data = nullptr;
other.m_rows = other.m_cols = 0;
other.m_dtype = Dtype(Dtype::TypeIndex::ERROR);
}
Frame Frame::clone() const {
Frame frame(m_rows, m_cols, m_dtype);
std::memcpy(frame.m_data, m_data, m_rows * m_cols * m_dtype.bytes());
return frame;
}
} // namespace aare

152
src/Frame.test.cpp Normal file
View File

@ -0,0 +1,152 @@
#include "aare/Frame.hpp"
#include "aare/Dtype.hpp"
#include <catch2/catch_test_macros.hpp>
using namespace aare;
TEST_CASE("Construct a frame") {
size_t rows = 10;
size_t cols = 10;
size_t bitdepth = 8;
Frame frame(rows, cols, Dtype::from_bitdepth(bitdepth));
REQUIRE(frame.rows() == rows);
REQUIRE(frame.cols() == cols);
REQUIRE(frame.bitdepth() == bitdepth);
REQUIRE(frame.bytes() == rows * cols * bitdepth / 8);
// data should be initialized to 0
for (size_t i = 0; i < rows; i++) {
for (size_t j = 0; j < cols; j++) {
uint8_t *data = (uint8_t *)frame.pixel_ptr(i, j);
REQUIRE(data != nullptr);
REQUIRE(*data == 0);
}
}
}
TEST_CASE("Set a value in a 8 bit frame") {
size_t rows = 10;
size_t cols = 10;
size_t bitdepth = 8;
Frame frame(rows, cols, Dtype::from_bitdepth(bitdepth));
// set a value
uint8_t value = 255;
frame.set(5, 7, value);
// only the value we did set should be non-zero
for (size_t i = 0; i < rows; i++) {
for (size_t j = 0; j < cols; j++) {
uint8_t *data = (uint8_t *)frame.pixel_ptr(i, j);
REQUIRE(data != nullptr);
if (i == 5 && j == 7) {
REQUIRE(*data == value);
} else {
REQUIRE(*data == 0);
}
}
}
}
TEST_CASE("Set a value in a 64 bit frame") {
size_t rows = 10;
size_t cols = 10;
size_t bitdepth = 64;
Frame frame(rows, cols, Dtype::from_bitdepth(bitdepth));
// set a value
uint64_t value = 255;
frame.set(5, 7, value);
// only the value we did set should be non-zero
for (size_t i = 0; i < rows; i++) {
for (size_t j = 0; j < cols; j++) {
uint64_t *data = (uint64_t *)frame.pixel_ptr(i, j);
REQUIRE(data != nullptr);
if (i == 5 && j == 7) {
REQUIRE(*data == value);
} else {
REQUIRE(*data == 0);
}
}
}
}
TEST_CASE("Move construct a frame") {
size_t rows = 10;
size_t cols = 10;
size_t bitdepth = 8;
Frame frame(rows, cols, Dtype::from_bitdepth(bitdepth));
std::byte *data = frame.data();
Frame frame2(std::move(frame));
// state of the moved from object
REQUIRE(frame.rows() == 0);
REQUIRE(frame.cols() == 0);
REQUIRE(frame.dtype() == Dtype(Dtype::TypeIndex::ERROR));
REQUIRE(frame.data() == nullptr);
// state of the moved to object
REQUIRE(frame2.rows() == rows);
REQUIRE(frame2.cols() == cols);
REQUIRE(frame2.bitdepth() == bitdepth);
REQUIRE(frame2.bytes() == rows * cols * bitdepth / 8);
REQUIRE(frame2.data() == data);
}
TEST_CASE("Move assign a frame") {
size_t rows = 10;
size_t cols = 10;
size_t bitdepth = 8;
Frame frame(rows, cols, Dtype::from_bitdepth(bitdepth));
std::byte *data = frame.data();
Frame frame2(5, 5, Dtype::from_bitdepth(16));
frame2 = std::move(frame);
// state of the moved from object
REQUIRE(frame.rows() == 0);
REQUIRE(frame.cols() == 0);
REQUIRE(frame.dtype() == Dtype(Dtype::TypeIndex::ERROR));
REQUIRE(frame.data() == nullptr);
// state of the moved to object
REQUIRE(frame2.rows() == rows);
REQUIRE(frame2.cols() == cols);
REQUIRE(frame2.bitdepth() == bitdepth);
REQUIRE(frame2.bytes() == rows * cols * bitdepth / 8);
REQUIRE(frame2.data() == data);
}
TEST_CASE("test explicit copy constructor") {
size_t rows = 10;
size_t cols = 10;
size_t bitdepth = 8;
Frame frame(rows, cols, Dtype::from_bitdepth(bitdepth));
std::byte *data = frame.data();
Frame frame2 = frame.clone();
// state of the original object
REQUIRE(frame.rows() == rows);
REQUIRE(frame.cols() == cols);
REQUIRE(frame.bitdepth() == bitdepth);
REQUIRE(frame.bytes() == rows * cols * bitdepth / 8);
REQUIRE(frame.data() == data);
// state of the copied object
REQUIRE(frame2.rows() == rows);
REQUIRE(frame2.cols() == cols);
REQUIRE(frame2.bitdepth() == bitdepth);
REQUIRE(frame2.bytes() == rows * cols * bitdepth / 8);
REQUIRE(frame2.data() != data);
}

377
src/NDArray.test.cpp Normal file
View File

@ -0,0 +1,377 @@
#include "aare/NDArray.hpp"
#include <array>
#include <catch2/catch_test_macros.hpp>
using aare::NDArray;
using aare::NDView;
using aare::Shape;
TEST_CASE("Initial size is zero if no size is specified") {
NDArray<double> a;
REQUIRE(a.size() == 0);
REQUIRE(a.shape() == Shape<2>{0, 0});
}
TEST_CASE("Construct from a DataSpan") {
std::vector<int> some_data(9, 42);
NDView<int, 2> view(some_data.data(), Shape<2>{3, 3});
NDArray<int, 2> image(view);
REQUIRE(image.shape() == view.shape());
REQUIRE(image.size() == view.size());
REQUIRE(image.data() != view.data());
for (uint32_t i = 0; i < image.size(); ++i) {
REQUIRE(image(i) == view(i));
}
// Changing the image doesn't change the view
image = 43;
for (uint32_t i = 0; i < image.size(); ++i) {
REQUIRE(image(i) != view(i));
}
}
TEST_CASE("1D image") {
std::array<int64_t, 1> shape{{20}};
NDArray<short, 1> img(shape, 3);
REQUIRE(img.size() == 20);
REQUIRE(img(5) == 3);
}
TEST_CASE("Accessing a const object") {
const NDArray<double, 3> img({3, 4, 5}, 0);
REQUIRE(img(1, 1, 1) == 0);
REQUIRE(img.size() == 3 * 4 * 5);
REQUIRE(img.shape() == Shape<3>{3, 4, 5});
REQUIRE(img.shape(0) == 3);
REQUIRE(img.shape(1) == 4);
REQUIRE(img.shape(2) == 5);
}
TEST_CASE("Indexing of a 2D image") {
std::array<int64_t, 2> shape{{3, 7}};
NDArray<long> img(shape, 5);
for (uint32_t i = 0; i != img.size(); ++i) {
REQUIRE(img(i) == 5);
}
for (uint32_t i = 0; i != img.size(); ++i) {
img(i) = i;
}
REQUIRE(img(0, 0) == 0);
REQUIRE(img(0, 1) == 1);
REQUIRE(img(1, 0) == 7);
}
TEST_CASE("Indexing of a 3D image") {
NDArray<float, 3> img{{{3, 4, 2}}, 5.0f};
for (uint32_t i = 0; i != img.size(); ++i) {
REQUIRE(img(i) == 5.0f);
}
// Double check general properties
REQUIRE(img.size() == 3 * 4 * 2);
for (uint32_t i = 0; i != img.size(); ++i) {
img(i) = float(i);
}
REQUIRE(img(0, 0, 0) == 0);
REQUIRE(img(0, 0, 1) == 1);
REQUIRE(img(0, 1, 1) == 3);
REQUIRE(img(1, 2, 0) == 12);
REQUIRE(img(2, 3, 1) == 23);
}
TEST_CASE("Divide double by int") {
NDArray<double, 1> a{{5}, 5};
NDArray<int, 1> b{{5}, 5};
a /= b;
for (auto it : a) {
REQUIRE(it == 1.0);
}
}
TEST_CASE("Elementwise multiplication of 3D image") {
std::array<int64_t, 3> shape{3, 4, 2};
NDArray<double, 3> a{shape};
NDArray<double, 3> b{shape};
for (uint32_t i = 0; i != a.size(); ++i) {
a(i) = i;
b(i) = i;
}
auto c = a * b;
REQUIRE(c(0, 0, 0) == 0 * 0);
REQUIRE(c(0, 0, 1) == 1 * 1);
REQUIRE(c(0, 1, 1) == 3 * 3);
REQUIRE(c(1, 2, 0) == 12 * 12);
REQUIRE(c(2, 3, 1) == 23 * 23);
}
TEST_CASE("Compare two images") {
NDArray<int> a;
NDArray<int> b;
CHECK((a == b));
a = NDArray<int>{{5, 10}, 0};
CHECK((a != b));
b = NDArray<int>{{5, 10}, 0};
CHECK((a == b));
b(3, 3) = 7;
CHECK((a != b));
}
TEST_CASE("Size and shape matches") {
int64_t w = 15;
int64_t h = 75;
std::array<int64_t, 2> shape{w, h};
NDArray<double> a{shape};
REQUIRE(a.size() == static_cast<uint64_t>(w * h));
REQUIRE(a.shape() == shape);
}
TEST_CASE("Initial value matches for all elements") {
double v = 4.35;
NDArray<double> a{{5, 5}, v};
for (uint32_t i = 0; i < a.size(); ++i) {
REQUIRE(a(i) == v);
}
}
TEST_CASE("Data layout of 3D image, fast index last") {
NDArray<int, 3> a{{3, 3, 3}, 0};
REQUIRE(a.size() == 27);
int *ptr = a.data();
for (int i = 0; i < 9; ++i) {
*ptr++ = 10 + i;
REQUIRE(a(0, 0, i) == 10 + i);
REQUIRE(a(i) == 10 + i);
}
}
TEST_CASE("Bitwise and on data") {
NDArray<uint16_t, 1> a({3}, 0);
uint16_t mask = 0x3FF;
a(0) = 16684;
a(1) = 33068;
a(2) = 52608;
a &= mask;
REQUIRE(a(0) == 300);
REQUIRE(a(1) == 300);
REQUIRE(a(2) == 384);
}
// TEST_CASE("Benchmarks")
// {
// NDArray<double> img;
// std::array<int64_t, 2> shape{ 512, 1024 };
// BENCHMARK("Allocate 500k double image")
// {
// NDArray<double>im{ shape };
// }
// BENCHMARK("Allocate 500k double image with initial value")
// {
// NDArray<double>im{ shape, 3.14 };
// }
// NDArray<double> a{ shape, 1.2 };
// NDArray<double> b{ shape, 53. };
// auto c = a + b;
// c = a * b;
// BENCHMARK("Multiply two images")
// {
// c = a * b;
// }
// BENCHMARK("Divide two images")
// {
// c = a / b;
// }
// BENCHMARK("Add two images")
// {
// c = a + b;
// }
// BENCHMARK("Subtract two images")
// {
// c = a - b;
// }
// }
TEST_CASE("Elementwise operatios on images") {
std::array<int64_t, 2> shape{5, 5};
double a_val = 3.0;
double b_val = 8.0;
SECTION("Add two images") {
NDArray<double> A(shape, a_val);
NDArray<double> B(shape, b_val);
auto C = A + B;
// Value of C matches
for (uint32_t i = 0; i < C.size(); ++i) {
REQUIRE(C(i) == a_val + b_val);
}
// Value of A is not changed
for (uint32_t i = 0; i < A.size(); ++i) {
REQUIRE(A(i) == a_val);
}
// Value of B is not changed
for (uint32_t i = 0; i < B.size(); ++i) {
REQUIRE(B(i) == b_val);
}
// A, B and C referes to different data
REQUIRE(A.data() != B.data());
REQUIRE(B.data() != C.data());
}
SECTION("Subtract two images") {
NDArray<double> A(shape, a_val);
NDArray<double> B(shape, b_val);
auto C = A - B;
// Value of C matches
for (uint32_t i = 0; i < C.size(); ++i) {
REQUIRE(C(i) == a_val - b_val);
}
// Value of A is not changed
for (uint32_t i = 0; i < A.size(); ++i) {
REQUIRE(A(i) == a_val);
}
// Value of B is not changed
for (uint32_t i = 0; i < B.size(); ++i) {
REQUIRE(B(i) == b_val);
}
// A, B and C referes to different data
REQUIRE(A.data() != B.data());
REQUIRE(B.data() != C.data());
}
SECTION("Multiply two images") {
NDArray<double> A(shape, a_val);
NDArray<double> B(shape, b_val);
auto C = A * B;
// Value of C matches
for (uint32_t i = 0; i < C.size(); ++i) {
REQUIRE(C(i) == a_val * b_val);
}
// Value of A is not changed
for (uint32_t i = 0; i < A.size(); ++i) {
REQUIRE(A(i) == a_val);
}
// Value of B is not changed
for (uint32_t i = 0; i < B.size(); ++i) {
REQUIRE(B(i) == b_val);
}
// A, B and C referes to different data
REQUIRE(A.data() != B.data());
REQUIRE(B.data() != C.data());
}
SECTION("Divide two images") {
NDArray<double> A(shape, a_val);
NDArray<double> B(shape, b_val);
auto C = A / B;
// Value of C matches
for (uint32_t i = 0; i < C.size(); ++i) {
REQUIRE(C(i) == a_val / b_val);
}
// Value of A is not changed
for (uint32_t i = 0; i < A.size(); ++i) {
REQUIRE(A(i) == a_val);
}
// Value of B is not changed
for (uint32_t i = 0; i < B.size(); ++i) {
REQUIRE(B(i) == b_val);
}
// A, B and C referes to different data
REQUIRE(A.data() != B.data());
REQUIRE(B.data() != C.data());
}
SECTION("subtract scalar") {
NDArray<double> A(shape, a_val);
NDArray<double> B(shape, b_val);
double v = 1.0;
auto C = A - v;
REQUIRE(C.data() != A.data());
// Value of C matches
for (uint32_t i = 0; i < C.size(); ++i) {
REQUIRE(C(i) == a_val - v);
}
// Value of A is not changed
for (uint32_t i = 0; i < A.size(); ++i) {
REQUIRE(A(i) == a_val);
}
}
SECTION("add scalar") {
NDArray<double> A(shape, a_val);
NDArray<double> B(shape, b_val);
double v = 1.0;
auto C = A + v;
REQUIRE(C.data() != A.data());
// Value of C matches
for (uint32_t i = 0; i < C.size(); ++i) {
REQUIRE(C(i) == a_val + v);
}
// Value of A is not changed
for (uint32_t i = 0; i < A.size(); ++i) {
REQUIRE(A(i) == a_val);
}
}
SECTION("divide with scalar") {
NDArray<double> A(shape, a_val);
NDArray<double> B(shape, b_val);
double v = 3.7;
auto C = A / v;
REQUIRE(C.data() != A.data());
// Value of C matches
for (uint32_t i = 0; i < C.size(); ++i) {
REQUIRE(C(i) == a_val / v);
}
// Value of A is not changed
for (uint32_t i = 0; i < A.size(); ++i) {
REQUIRE(A(i) == a_val);
}
}
SECTION("multiply with scalar") {
NDArray<double> A(shape, a_val);
NDArray<double> B(shape, b_val);
double v = 3.7;
auto C = A / v;
REQUIRE(C.data() != A.data());
// Value of C matches
for (uint32_t i = 0; i < C.size(); ++i) {
REQUIRE(C(i) == a_val / v);
}
// Value of A is not changed
for (uint32_t i = 0; i < A.size(); ++i) {
REQUIRE(A(i) == a_val);
}
}
}

193
src/NDView.test.cpp Normal file
View File

@ -0,0 +1,193 @@
#include "aare/NDView.hpp"
#include <catch2/catch_test_macros.hpp>
#include <iostream>
#include <vector>
using aare::NDView;
using aare::Shape;
TEST_CASE("Element reference 1D") {
std::vector<int> vec;
for (int i = 0; i != 10; ++i) {
vec.push_back(i);
}
NDView<int, 1> data(vec.data(), Shape<1>{10});
REQUIRE(vec.size() == static_cast<size_t>(data.size()));
for (int i = 0; i != 10; ++i) {
REQUIRE(data(i) == vec[i]);
REQUIRE(data[i] == vec[i]);
}
}
TEST_CASE("Element reference 2D") {
std::vector<int> vec;
for (int i = 0; i != 12; ++i) {
vec.push_back(i);
}
NDView<int, 2> data(vec.data(), Shape<2>{3, 4});
REQUIRE(vec.size() == static_cast<size_t>(data.size()));
int i = 0;
for (int row = 0; row != 3; ++row) {
for (int col = 0; col != 4; ++col) {
REQUIRE(data(row, col) == i);
REQUIRE(data[i] == vec[i]);
++i;
}
}
}
TEST_CASE("Element reference 3D") {
std::vector<int> vec;
for (int i = 0; i != 24; ++i) {
vec.push_back(i);
}
NDView<int, 3> data(vec.data(), Shape<3>{2, 3, 4});
REQUIRE(vec.size() == static_cast<size_t>(data.size()));
int i = 0;
for (int frame = 0; frame != 2; ++frame) {
for (int row = 0; row != 3; ++row) {
for (int col = 0; col != 4; ++col) {
REQUIRE(data(frame, row, col) == i);
REQUIRE(data[i] == vec[i]);
++i;
}
}
}
}
TEST_CASE("Plus and miuns with single value") {
std::vector<int> vec;
for (int i = 0; i != 12; ++i) {
vec.push_back(i);
}
NDView<int, 2> data(vec.data(), Shape<2>{3, 4});
data += 5;
int i = 0;
for (int row = 0; row != 3; ++row) {
for (int col = 0; col != 4; ++col) {
REQUIRE(data(row, col) == i + 5);
++i;
}
}
data -= 3;
i = 0;
for (int row = 0; row != 3; ++row) {
for (int col = 0; col != 4; ++col) {
REQUIRE(data(row, col) == i + 2);
++i;
}
}
}
TEST_CASE("Multiply and divide with single value") {
std::vector<int> vec;
for (int i = 0; i != 12; ++i) {
vec.push_back(i);
}
NDView<int, 2> data(vec.data(), Shape<2>{3, 4});
data *= 5;
int i = 0;
for (int row = 0; row != 3; ++row) {
for (int col = 0; col != 4; ++col) {
REQUIRE(data(row, col) == i * 5);
++i;
}
}
data /= 3;
i = 0;
for (int row = 0; row != 3; ++row) {
for (int col = 0; col != 4; ++col) {
REQUIRE(data(row, col) == (i * 5) / 3);
++i;
}
}
}
TEST_CASE("elementwise assign") {
std::vector<int> vec(25);
NDView<int, 2> data(vec.data(), Shape<2>{5, 5});
data = 3;
for (auto it : data) {
REQUIRE(it == 3);
}
}
TEST_CASE("iterators") {
std::vector<int> vec;
for (int i = 0; i != 12; ++i) {
vec.push_back(i);
}
NDView<int, 1> data(vec.data(), Shape<1>{12});
int i = 0;
for (const auto item : data) {
REQUIRE(item == vec[i]);
++i;
}
REQUIRE(i == 12);
for (auto ptr = data.begin(); ptr != data.end(); ++ptr) {
*ptr += 1;
}
for (auto &item : data) {
++item;
}
i = 0;
for (const auto item : data) {
REQUIRE(item == i + 2);
++i;
}
}
// TEST_CASE("shape from vector") {
// std::vector<int> vec;
// for (int i = 0; i != 12; ++i) {
// vec.push_back(i);
// }
// std::vector<int64_t> shape{3, 4};
// NDView<int, 2> data(vec.data(), shape);
// }
TEST_CASE("divide with another span") {
std::vector<int> vec0{9, 12, 3};
std::vector<int> vec1{3, 2, 1};
std::vector<int> result{3, 6, 3};
NDView<int, 1> data0(vec0.data(), Shape<1>{static_cast<int64_t>(vec0.size())});
NDView<int, 1> data1(vec1.data(), Shape<1>{static_cast<int64_t>(vec1.size())});
data0 /= data1;
for (size_t i = 0; i != vec0.size(); ++i) {
REQUIRE(data0[i] == result[i]);
}
}
TEST_CASE("Retrieve shape") {
std::vector<int> vec;
for (int i = 0; i != 12; ++i) {
vec.push_back(i);
}
NDView<int, 2> data(vec.data(), Shape<2>{3, 4});
REQUIRE(data.shape()[0] == 3);
REQUIRE(data.shape()[1] == 4);
}
TEST_CASE("compare two views") {
std::vector<int> vec1;
for (int i = 0; i != 12; ++i) {
vec1.push_back(i);
}
NDView<int, 2> view1(vec1.data(), Shape<2>{3, 4});
std::vector<int> vec2;
for (int i = 0; i != 12; ++i) {
vec2.push_back(i);
}
NDView<int, 2> view2(vec2.data(), Shape<2>{3, 4});
REQUIRE((view1 == view2));
}

200
src/NumpyFile.cpp Normal file
View File

@ -0,0 +1,200 @@
#include "aare/NumpyFile.hpp"
#include "aare/NumpyHelpers.hpp"
namespace aare {
NumpyFile::NumpyFile(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg) {
// TODO! add opts to constructor
m_mode = mode;
if (mode == "r") {
fp = fopen(fname.string().c_str(), "rb");
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));
}
load_metadata();
} else if (mode == "w") {
m_bitdepth = cfg.dtype.bitdepth();
m_rows = cfg.rows;
m_cols = cfg.cols;
m_header = {cfg.dtype, false, {cfg.rows, cfg.cols}};
m_header.shape = {0, cfg.rows, cfg.cols};
fp = fopen(fname.string().c_str(), "wb");
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));
}
initial_header_len = aare::NumpyHelpers::write_header(std::filesystem::path(fname.c_str()), m_header);
}
m_pixels_per_frame = std::accumulate(m_header.shape.begin() + 1, m_header.shape.end(), 1, std::multiplies<>());
m_bytes_per_frame = m_header.dtype.bitdepth() / 8 * m_pixels_per_frame;
}
void NumpyFile::write(Frame &frame) { write_impl(frame.data(), frame.bytes()); }
void NumpyFile::write_impl(void *data, uint64_t size) {
if (fp == nullptr) {
throw std::runtime_error("File not open");
}
if (!(m_mode == "w" || m_mode == "a")) {
throw std::invalid_argument("File not open for writing");
}
if (fseek(fp, 0, SEEK_END))
throw std::runtime_error("Could not seek to end of file");
size_t const rc = fwrite(data, size, 1, fp);
if (rc != 1) {
throw std::runtime_error("Error writing frame to file");
}
m_header.shape[0]++;
}
Frame NumpyFile::get_frame(size_t frame_number) {
Frame frame(m_header.shape[1], m_header.shape[2], m_header.dtype);
get_frame_into(frame_number, frame.data());
return frame;
}
void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) {
if (fp == nullptr) {
throw std::runtime_error("File not open");
}
if (frame_number > m_header.shape[0]) {
throw std::invalid_argument("Frame number out of range");
}
if (fseek(fp, header_size + frame_number * m_bytes_per_frame, SEEK_SET)) // NOLINT
throw std::runtime_error("Could not seek to frame");
size_t const rc = fread(image_buf, m_bytes_per_frame, 1, fp);
if (rc != 1) {
throw std::runtime_error("Error reading frame from file");
}
}
size_t NumpyFile::pixels_per_frame() { return m_pixels_per_frame; };
size_t NumpyFile::bytes_per_frame() { return m_bytes_per_frame; };
std::vector<Frame> NumpyFile::read_n(size_t n_frames) {
// TODO: implement this in a more efficient way
std::vector<Frame> frames;
for (size_t i = 0; i < n_frames; i++) {
frames.push_back(get_frame(current_frame));
current_frame++;
}
return frames;
}
void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) {
// TODO: implement this in a more efficient way
for (size_t i = 0; i < n_frames; i++) {
get_frame_into(current_frame++, image_buf);
image_buf += m_bytes_per_frame;
}
}
NumpyFile::~NumpyFile() noexcept {
if (m_mode == "w" || m_mode == "a") {
// determine number of frames
if (fseek(fp, 0, SEEK_END)) {
std::cout << "Could not seek to end of file" << std::endl;
}
size_t const file_size = ftell(fp);
size_t const data_size = file_size - initial_header_len;
size_t const n_frames = data_size / m_bytes_per_frame;
// update number of frames in header (first element of shape)
m_header.shape[0] = n_frames;
if (fseek(fp, 0, SEEK_SET)) {
std::cout << "Could not seek to beginning of file" << std::endl;
}
// create string stream to contain header
std::stringstream ss;
aare::NumpyHelpers::write_header(ss, m_header);
std::string const header_str = ss.str();
// write header
size_t const rc = fwrite(header_str.c_str(), header_str.size(), 1, fp);
if (rc != 1) {
std::cout << "Error writing header to numpy file in destructor" << std::endl;
}
}
if (fp != nullptr) {
if (fclose(fp)) {
std::cout << "Error closing file" << std::endl;
}
}
}
void NumpyFile::load_metadata() {
// read magic number
std::array<char, 6> tmp{};
size_t rc = fread(tmp.data(), tmp.size(), 1, fp);
if (rc != 1) {
throw std::runtime_error("Error reading magic number");
}
if (tmp != aare::NumpyHelpers::magic_str) {
for (auto item : tmp)
fmt::print("{}, ", static_cast<int>(item));
fmt::print("\n");
throw std::runtime_error("Not a numpy file");
}
// read version
rc = fread(reinterpret_cast<char *>(&major_ver_), sizeof(major_ver_), 1, fp);
rc += fread(reinterpret_cast<char *>(&minor_ver_), sizeof(minor_ver_), 1, fp);
if (rc != 2) {
throw std::runtime_error("Error reading numpy version");
}
if (major_ver_ == 1) {
header_len_size = 2;
} else if (major_ver_ == 2) {
header_len_size = 4;
} else {
throw std::runtime_error("Unsupported numpy version");
}
// read header length
rc = fread(reinterpret_cast<char *>(&header_len), header_len_size, 1, fp);
if (rc != 1) {
throw std::runtime_error("Error reading header length");
}
header_size = aare::NumpyHelpers::magic_string_length + 2 + header_len_size + header_len;
if (header_size % 16 != 0) {
fmt::print("Warning: header length is not a multiple of 16\n");
}
// read header
std::string header(header_len, '\0');
rc = fread(header.data(), header_len, 1, fp);
if (rc != 1) {
throw std::runtime_error("Error reading header");
}
// parse header
std::vector<std::string> const keys{"descr", "fortran_order", "shape"};
auto dict_map = aare::NumpyHelpers::parse_dict(header, keys);
if (dict_map.empty())
throw std::runtime_error("invalid dictionary in header");
std::string const descr_s = dict_map["descr"];
std::string const fortran_s = dict_map["fortran_order"];
std::string const shape_s = dict_map["shape"];
std::string const descr = aare::NumpyHelpers::parse_str(descr_s);
aare::Dtype const dtype = aare::NumpyHelpers::parse_descr(descr);
// convert literal Python bool to C++ bool
bool const fortran_order = aare::NumpyHelpers::parse_bool(fortran_s);
// parse the shape tuple
auto shape_v = aare::NumpyHelpers::parse_tuple(shape_s);
std::vector<size_t> shape;
for (const auto &item : shape_v) {
auto dim = static_cast<size_t>(std::stoul(item));
shape.push_back(dim);
}
m_header = {dtype, fortran_order, shape};
}
} // namespace aare

50
src/NumpyFile.test.cpp Normal file
View File

@ -0,0 +1,50 @@
#include "aare/NumpyFile.hpp"
#include "aare/NDArray.hpp"
#include <catch2/catch_test_macros.hpp>
#include "test_config.hpp"
using aare::Dtype;
using aare::NumpyFile;
TEST_CASE("Read a 1D numpy file with int32 data type") {
auto fpath = test_data_path() / "numpy" / "test_1d_int32.npy";
REQUIRE(std::filesystem::exists(fpath));
NumpyFile f(fpath);
// we know the file contains 10 elements of np.int32 containing values 0-9
REQUIRE(f.dtype() == Dtype::INT32);
REQUIRE(f.shape() == std::vector<size_t>{10});
// use the load function to read the full file into a NDArray
auto data = f.load<int32_t, 1>();
for (int32_t i = 0; i < 10; i++) {
REQUIRE(data(i) == i);
}
}
TEST_CASE("Read a 3D numpy file with np.double data type") {
auto fpath = test_data_path() / "numpy" / "test_3d_double.npy";
REQUIRE(std::filesystem::exists(fpath));
NumpyFile f(fpath);
// we know the file contains 10 elements of np.int32 containing values 0-9
REQUIRE(f.dtype() == Dtype::DOUBLE);
REQUIRE(f.shape() == std::vector<size_t>{3, 2, 5});
// use the load function to read the full file into a NDArray
// numpy code to generate the array
// arr2[0,0,0] = 1.0
// arr2[0,0,1] = 2.0
// arr2[0,1,0] = 72.0
// arr2[2,0,4] = 63.0
auto data = f.load<double, 3>();
REQUIRE(data(0, 0, 0) == 1.0);
REQUIRE(data(0, 0, 1) == 2.0);
REQUIRE(data(0, 1, 0) == 72.0);
REQUIRE(data(2, 0, 4) == 63.0);
}

269
src/NumpyHelpers.cpp Normal file
View File

@ -0,0 +1,269 @@
/*
28-03-2024 modified by: Bechir Braham <bechir.braham@psi.ch>
Copyright 2017-2023 Leon Merten Lohse
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#include "aare/NumpyHelpers.hpp"
#include <iterator>
namespace aare {
std::string NumpyHeader::to_string() const {
std::stringstream sstm;
sstm << "dtype: " << dtype.to_string() << ", fortran_order: " << fortran_order << ' ';
sstm << "shape: (";
for (auto item : shape)
sstm << item << ',';
sstm << ')';
return sstm.str();
}
namespace NumpyHelpers {
std::unordered_map<std::string, std::string> parse_dict(std::string in, const std::vector<std::string> &keys) {
std::unordered_map<std::string, std::string> map;
if (keys.empty())
return map;
in = trim(in);
// unwrap dictionary
if ((in.front() == '{') && (in.back() == '}'))
in = in.substr(1, in.length() - 2);
else
throw std::runtime_error("Not a Python dictionary.");
std::vector<std::pair<size_t, std::string>> positions;
for (auto const &key : keys) {
size_t const pos = in.find("'" + key + "'");
if (pos == std::string::npos)
throw std::runtime_error("Missing '" + key + "' key.");
std::pair<size_t, std::string> const position_pair{pos, key};
positions.push_back(position_pair);
}
// sort by position in dict
std::sort(positions.begin(), positions.end());
for (size_t i = 0; i < positions.size(); ++i) {
std::string raw_value;
size_t const begin{positions[i].first};
size_t end{std::string::npos};
std::string const key = positions[i].second;
if (i + 1 < positions.size())
end = positions[i + 1].first;
raw_value = in.substr(begin, end - begin);
raw_value = trim(raw_value);
if (raw_value.back() == ',')
raw_value.pop_back();
map[key] = get_value_from_map(raw_value);
}
return map;
}
aare::Dtype parse_descr(std::string typestring) {
if (typestring.length() < 3) {
throw std::runtime_error("invalid typestring (length)");
}
constexpr char little_endian_char = '<';
constexpr char big_endian_char = '>';
constexpr char no_endian_char = '|';
constexpr std::array<char, 3> endian_chars = {little_endian_char, big_endian_char, no_endian_char};
constexpr std::array<char, 4> numtype_chars = {'f', 'i', 'u', 'c'};
const char byteorder_c = typestring[0];
const char kind_c = typestring[1];
std::string const itemsize_s = typestring.substr(2);
if (!in_array(byteorder_c, endian_chars)) {
throw std::runtime_error("invalid typestring (byteorder)");
}
if (!in_array(kind_c, numtype_chars)) {
throw std::runtime_error("invalid typestring (kind)");
}
if (!is_digits(itemsize_s)) {
throw std::runtime_error("invalid typestring (itemsize)");
}
return aare::Dtype(typestring);
}
bool parse_bool(const std::string &in) {
if (in == "True")
return true;
if (in == "False")
return false;
throw std::runtime_error("Invalid python boolean.");
}
std::string get_value_from_map(const std::string &mapstr) {
size_t const sep_pos = mapstr.find_first_of(':');
if (sep_pos == std::string::npos)
return "";
std::string const tmp = mapstr.substr(sep_pos + 1);
return trim(tmp);
}
bool is_digits(const std::string &str) { return std::all_of(str.begin(), str.end(), ::isdigit); }
std::vector<std::string> parse_tuple(std::string in) {
std::vector<std::string> v;
const char separator = ',';
in = trim(in);
if ((in.front() == '(') && (in.back() == ')'))
in = in.substr(1, in.length() - 2);
else
throw std::runtime_error("Invalid Python tuple.");
std::istringstream iss(in);
for (std::string token; std::getline(iss, token, separator);) {
v.push_back(token);
}
return v;
}
std::string trim(const std::string &str) {
const std::string whitespace = " \t\n";
auto begin = str.find_first_not_of(whitespace);
if (begin == std::string::npos)
return "";
auto end = str.find_last_not_of(whitespace);
return str.substr(begin, end - begin + 1);
}
std::string parse_str(const std::string &in) {
if ((in.front() == '\'') && (in.back() == '\''))
return in.substr(1, in.length() - 2);
throw std::runtime_error("Invalid python string.");
}
void write_magic(std::ostream &ostream, int version_major, int version_minor) {
ostream.write(magic_str.data(), magic_string_length);
ostream.put(static_cast<char>(version_major));
ostream.put(static_cast<char>(version_minor));
}
template <typename T> inline std::string write_tuple(const std::vector<T> &v) {
if (v.empty())
return "()";
std::ostringstream ss;
ss.imbue(std::locale("C"));
if (v.size() == 1) {
ss << "(" << v.front() << ",)";
} else {
const std::string delimiter = ", ";
// v.size() > 1
ss << "(";
// for (size_t i = 0; i < v.size() - 1; ++i) {
// ss << v[i] << delimiter;
// }
// ss << v.back();
std::copy(v.begin(), v.end() - 1, std::ostream_iterator<T>(ss, ", "));
ss << v.back();
ss << ")";
}
return ss.str();
}
inline std::string write_boolean(bool b) {
if (b)
return "True";
return "False";
}
inline std::string write_header_dict(const std::string &descr, bool fortran_order, const std::vector<size_t> &shape) {
std::string const s_fortran_order = write_boolean(fortran_order);
std::string const shape_s = write_tuple(shape);
return "{'descr': '" + descr + "', 'fortran_order': " + s_fortran_order + ", 'shape': " + shape_s + ", }";
}
size_t write_header(const std::filesystem::path &fname, const NumpyHeader &header) {
std::ofstream out(fname, std::ios::binary | std::ios::out);
return write_header(out, header);
}
size_t write_header(std::ostream &out, const NumpyHeader &header) {
std::string const header_dict = write_header_dict(header.dtype.to_string(), header.fortran_order, header.shape);
size_t length = magic_string_length + 2 + 2 + header_dict.length() + 1;
int version_major = 1;
int version_minor = 0;
if (length >= static_cast<size_t>(255) * 255) {
length = magic_string_length + 2 + 4 + header_dict.length() + 1;
version_major = 2;
version_minor = 0;
}
size_t const padding_len = 16 - length % 16;
std::string const padding(padding_len, ' ');
// write magic
write_magic(out, version_major, version_minor);
// write header length
if (version_major == 1 && version_minor == 0) {
auto header_len = static_cast<uint16_t>(header_dict.length() + padding.length() + 1);
std::array<uint8_t, 2> header_len_le16{static_cast<uint8_t>((header_len >> 0) & 0xff),
static_cast<uint8_t>((header_len >> 8) & 0xff)};
out.write(reinterpret_cast<char *>(header_len_le16.data()), 2);
} else {
auto header_len = static_cast<uint32_t>(header_dict.length() + padding.length() + 1);
std::array<uint8_t, 4> header_len_le32{
static_cast<uint8_t>((header_len >> 0) & 0xff), static_cast<uint8_t>((header_len >> 8) & 0xff),
static_cast<uint8_t>((header_len >> 16) & 0xff), static_cast<uint8_t>((header_len >> 24) & 0xff)};
out.write(reinterpret_cast<char *>(header_len_le32.data()), 4);
}
out << header_dict << padding << '\n';
return length;
}
} // namespace NumpyHelpers
} // namespace aare

62
src/NumpyHelpers.test.cpp Normal file
View File

@ -0,0 +1,62 @@
#include "aare/NumpyHelpers.hpp" //Is this really a public header?
#include <catch2/catch_test_macros.hpp>
using namespace aare::NumpyHelpers;
TEST_CASE("is_digits with a few standard cases") {
REQUIRE(is_digits(""));
REQUIRE(is_digits("123"));
REQUIRE(is_digits("0"));
REQUIRE_FALSE(is_digits("hej123"));
REQUIRE_FALSE(is_digits("a"));
REQUIRE_FALSE(is_digits(" "));
REQUIRE_FALSE(is_digits("abcdef"));
}
TEST_CASE("Check for quotes and return stripped string") {
REQUIRE(parse_str("'hej'") == "hej");
REQUIRE(parse_str("'hej hej'") == "hej hej");
REQUIRE(parse_str("''") == "");
}
TEST_CASE("parsing a string without quotes throws") { REQUIRE_THROWS(parse_str("hej")); }
TEST_CASE("trim whitespace") {
REQUIRE(trim(" hej ") == "hej");
REQUIRE(trim("hej") == "hej");
REQUIRE(trim(" hej") == "hej");
REQUIRE(trim("hej ") == "hej");
REQUIRE(trim(" ") == "");
REQUIRE(trim(" \thej hej ") == "hej hej");
}
TEST_CASE("parse data type descriptions") {
REQUIRE(parse_descr("<i1") == aare::Dtype::INT8);
REQUIRE(parse_descr("<i2") == aare::Dtype::INT16);
REQUIRE(parse_descr("<i4") == aare::Dtype::INT32);
REQUIRE(parse_descr("<i8") == aare::Dtype::INT64);
REQUIRE(parse_descr("<u1") == aare::Dtype::UINT8);
REQUIRE(parse_descr("<u2") == aare::Dtype::UINT16);
REQUIRE(parse_descr("<u4") == aare::Dtype::UINT32);
REQUIRE(parse_descr("<u8") == aare::Dtype::UINT64);
REQUIRE(parse_descr("<f4") == aare::Dtype::FLOAT);
REQUIRE(parse_descr("<f8") == aare::Dtype::DOUBLE);
}
TEST_CASE("is element in array") {
REQUIRE(in_array(1, std::array<int, 3>{1, 2, 3}));
REQUIRE_FALSE(in_array(4, std::array<int, 3>{1, 2, 3}));
REQUIRE(in_array(1, std::array<int, 1>{1}));
REQUIRE_FALSE(in_array(1, std::array<int, 0>{}));
}
TEST_CASE("Parse numpy dict") {
std::string in = "{'descr': '<f4', 'fortran_order': False, 'shape': (3, 4)}";
std::vector<std::string> keys{"descr", "fortran_order", "shape"};
auto map = parse_dict(in, keys);
REQUIRE(map["descr"] == "'<f4'");
REQUIRE(map["fortran_order"] == "False");
REQUIRE(map["shape"] == "(3, 4)");
}

103
src/Pedestal.test.cpp Normal file
View File

@ -0,0 +1,103 @@
#include "aare/Pedestal.hpp"
#include <catch2/matchers/catch_matchers_floating_point.hpp>
#include <catch2/catch_test_macros.hpp>
#include <chrono>
#include <random>
using namespace aare;
TEST_CASE("test pedestal constructor") {
aare::Pedestal pedestal(10, 10, 5);
REQUIRE(pedestal.rows() == 10);
REQUIRE(pedestal.cols() == 10);
REQUIRE(pedestal.n_samples() == 5);
for (int i = 0; i < 10; i++) {
for (int j = 0; j < 10; j++) {
REQUIRE(pedestal.get_sum()(i, j) == 0);
REQUIRE(pedestal.get_sum2()(i, j) == 0);
REQUIRE(pedestal.cur_samples()(i, j) == 0);
}
}
}
TEST_CASE("test pedestal push") {
aare::Pedestal pedestal(10, 10, 5);
aare::Frame frame(10, 10, Dtype::UINT16);
for (int i = 0; i < 10; i++) {
for (int j = 0; j < 10; j++) {
frame.set<uint16_t>(i, j, i + j);
}
}
// test single push
pedestal.push<uint16_t>(frame);
for (int i = 0; i < 10; i++) {
for (int j = 0; j < 10; j++) {
REQUIRE(pedestal.get_sum()(i, j) == i + j);
REQUIRE(pedestal.get_sum2()(i, j) == (i + j) * (i + j));
REQUIRE(pedestal.cur_samples()(i, j) == 1);
}
}
// test clear
pedestal.clear();
for (int i = 0; i < 10; i++) {
for (int j = 0; j < 10; j++) {
REQUIRE(pedestal.get_sum()(i, j) == 0);
REQUIRE(pedestal.get_sum2()(i, j) == 0);
REQUIRE(pedestal.cur_samples()(i, j) == 0);
}
}
// test number of samples after multiple push
for (uint32_t k = 0; k < 50; k++) {
pedestal.push<uint16_t>(frame);
for (uint32_t i = 0; i < 10; i++) {
for (uint32_t j = 0; j < 10; j++) {
if (k < 5) {
REQUIRE(pedestal.cur_samples()(i, j) == k + 1);
REQUIRE(pedestal.get_sum()(i, j) == (k + 1) * (i + j));
REQUIRE(pedestal.get_sum2()(i, j) == (k + 1) * (i + j) * (i + j));
} else {
REQUIRE(pedestal.cur_samples()(i, j) == 5);
REQUIRE(pedestal.get_sum()(i, j) == 5 * (i + j));
REQUIRE(pedestal.get_sum2()(i, j) == 5 * (i + j) * (i + j));
}
REQUIRE(pedestal.mean(i, j) == (i + j));
REQUIRE(pedestal.variance(i, j) == 0);
REQUIRE(pedestal.std(i, j) == 0);
}
}
}
}
TEST_CASE("test pedestal with normal distribution") {
const double MEAN = 5.0, STD = 2.0, VAR = STD * STD, TOLERANCE = 0.1;
unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
std::default_random_engine generator(seed);
std::normal_distribution<double> distribution(MEAN, STD);
aare::Pedestal pedestal(3, 5, 10000);
for (int i = 0; i < 10000; i++) {
aare::Frame frame(3, 5, Dtype::DOUBLE);
for (int j = 0; j < 3; j++) {
for (int k = 0; k < 5; k++) {
frame.set<double>(j, k, distribution(generator));
}
}
pedestal.push<double>(frame);
}
auto mean = pedestal.mean();
auto variance = pedestal.variance();
auto standard_deviation = pedestal.std();
for (int i = 0; i < 3; i++) {
for (int j = 0; j < 5; j++) {
REQUIRE_THAT(mean(i, j), Catch::Matchers::WithinAbs(MEAN, MEAN * TOLERANCE));
REQUIRE_THAT(variance(i, j), Catch::Matchers::WithinAbs(VAR, VAR * TOLERANCE));
REQUIRE_THAT(standard_deviation(i, j), Catch::Matchers::WithinAbs(STD, STD * TOLERANCE));
}
}
}

77
src/PixelMap.cpp Normal file
View File

@ -0,0 +1,77 @@
#include "aare/PixelMap.hpp"
#include <array>
namespace aare {
NDArray<ssize_t, 2> GenerateMoench03PixelMap() {
std::array<int, 32> const adc_nr = {300, 325, 350, 375, 300, 325, 350, 375,
200, 225, 250, 275, 200, 225, 250, 275,
100, 125, 150, 175, 100, 125, 150, 175,
0, 25, 50, 75, 0, 25, 50, 75};
int const sc_width = 25;
int const nadc = 32;
int const pixels_per_sc = 5000;
NDArray<ssize_t, 2> order_map({400, 400});
int pixel = 0;
for (int i = 0; i != pixels_per_sc; ++i) {
for (int i_adc = 0; i_adc != nadc; ++i_adc) {
int const col = adc_nr[i_adc] + (i % sc_width);
int row = 0;
if ((i_adc / 4) % 2 == 0)
row = 199 - (i / sc_width);
else
row = 200 + (i / sc_width);
order_map(row, col) = pixel;
pixel++;
}
}
return order_map;
}
NDArray<ssize_t, 2> GenerateMoench05PixelMap() {
std::array<int, 3> adc_numbers = {9, 13, 1};
NDArray<ssize_t, 2> order_map({160, 150});
int n_pixel = 0;
for (int row = 0; row < 160; row++) {
for (int i_col = 0; i_col < 50; i_col++) {
n_pixel = row * 50 + i_col;
for (int i_sc = 0; i_sc < 3; i_sc++) {
int col = 50 * i_sc + i_col;
int adc_nr = adc_numbers[i_sc];
int i_analog = n_pixel * 32 + adc_nr;
// analog_frame[row * 150 + col] = analog_data[i_analog] & 0x3FFF;
order_map(row, col) = i_analog;
}
}
}
return order_map;
}
NDArray<ssize_t, 2>GenerateMH02SingleCounterPixelMap(){
NDArray<ssize_t, 2> order_map({48, 48});
for(int row = 0; row < 48; row++){
for(int col = 0; col < 48; col++){
order_map(row, col) = row*48 + col;
}
}
return order_map;
}
NDArray<ssize_t, 3> GenerateMH02FourCounterPixelMap(){
NDArray<ssize_t, 3> order_map({4, 48, 48});
for (int counter=0; counter<4; counter++){
for(int row = 0; row < 48; row++){
for(int col = 0; col < 48; col++){
order_map(counter, row, col) = counter*48*48 + row*48 + col;
}
}
}
return order_map;
}
} // namespace aare

266
src/RawFile.cpp Normal file
View File

@ -0,0 +1,266 @@
#include "aare/RawFile.hpp"
#include "aare/PixelMap.hpp"
#include "aare/defs.hpp"
#include <fmt/format.h>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
namespace aare {
RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode)
: m_master(fname) {
m_mode = mode;
if (mode == "r") {
find_number_of_subfiles();
n_subfile_parts = m_master.geometry().col * m_master.geometry().row;
find_geometry();
open_subfiles();
} else {
throw std::runtime_error(LOCATION +
"Unsupported mode. Can only read RawFiles.");
}
}
Frame RawFile::read_frame() { return get_frame(m_current_frame++); };
Frame RawFile::read_frame(size_t frame_number) {
seek(frame_number);
return read_frame();
}
void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
// TODO: implement this in a more efficient way
for (size_t i = 0; i < n_frames; i++) {
this->get_frame_into(m_current_frame++, image_buf);
image_buf += bytes_per_frame();
}
}
void RawFile::read_into(std::byte *image_buf) {
return get_frame_into(m_current_frame++, image_buf);
};
size_t RawFile::bytes_per_frame() {
return m_rows * m_cols * m_master.bitdepth() / 8;
}
size_t RawFile::pixels_per_frame() { return m_rows * m_cols; }
DetectorType RawFile::detector_type() const { return m_master.detector_type(); }
void RawFile::seek(size_t frame_index) {
// check if the frame number is greater than the total frames
// if frame_number == total_frames, then the next read will throw an error
if (frame_index > total_frames()) {
throw std::runtime_error(
fmt::format("frame number {} is greater than total frames {}",
frame_index, total_frames()));
}
m_current_frame = frame_index;
};
size_t RawFile::tell() { return m_current_frame; };
size_t RawFile::total_frames() const { return m_master.frames_in_file(); }
size_t RawFile::rows() const { return m_rows; }
size_t RawFile::cols() const { return m_cols; }
size_t RawFile::bitdepth() const { return m_master.bitdepth(); }
xy RawFile::geometry() { return m_master.geometry(); }
void RawFile::open_subfiles() {
if (m_mode == "r")
for (size_t i = 0; i != n_subfiles; ++i) {
auto v = std::vector<SubFile *>(n_subfile_parts);
for (size_t j = 0; j != n_subfile_parts; ++j) {
v[j] =
new SubFile(m_master.data_fname(j, i),
m_master.detector_type(), m_master.pixels_y(),
m_master.pixels_x(), m_master.bitdepth());
}
subfiles.push_back(v);
}
else {
throw std::runtime_error(LOCATION +
"Unsupported mode. Can only read RawFiles.");
}
}
DetectorHeader RawFile::read_header(const std::filesystem::path &fname) {
DetectorHeader h{};
FILE *fp = fopen(fname.string().c_str(), "r");
if (!fp)
throw std::runtime_error(
fmt::format("Could not open: {} for reading", fname.string()));
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
if (rc != 1)
throw std::runtime_error(LOCATION + "Could not read header from file");
if (fclose(fp)) {
throw std::runtime_error(LOCATION + "Could not close file");
}
return h;
}
bool RawFile::is_master_file(const std::filesystem::path &fpath) {
std::string const stem = fpath.stem().string();
return stem.find("_master_") != std::string::npos;
}
void RawFile::find_number_of_subfiles() {
int n_mod = 0;
while (std::filesystem::exists(m_master.data_fname(0, ++n_mod)))
;
n_subfiles = n_mod;
#ifdef AARE_VERBOSE
fmt::print("Found: {} subfiles\n", n_subfiles);
#endif
}
void RawFile::find_geometry() {
uint16_t r{};
uint16_t c{};
for (size_t i = 0; i < n_subfile_parts; i++) {
for (size_t j = 0; j != n_subfiles; ++j) {
auto h = this->read_header(m_master.data_fname(i, j));
r = std::max(r, h.row);
c = std::max(c, h.column);
positions.push_back({h.row, h.column});
}
}
r++;
c++;
m_rows = (r * m_master.pixels_y());
m_cols = (c * m_master.pixels_x());
m_rows += static_cast<size_t>((r - 1) * cfg.module_gap_row);
}
Frame RawFile::get_frame(size_t frame_index) {
auto f = Frame(m_rows, m_cols, Dtype::from_bitdepth(m_master.bitdepth()));
std::byte *frame_buffer = f.data();
get_frame_into(frame_index, frame_buffer);
return f;
}
void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
if (frame_index > total_frames()) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
std::vector<size_t> frame_numbers(n_subfile_parts);
std::vector<size_t> frame_indices(n_subfile_parts, frame_index);
if (n_subfile_parts != 1) {
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
auto subfile_id = frame_index / m_master.max_frames_per_file();
frame_numbers[part_idx] =
subfiles[subfile_id][part_idx]->frame_number(
frame_index % m_master.max_frames_per_file());
}
// 1. if frame number vector is the same break
while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(),
std::not_equal_to<>()) !=
frame_numbers.end()) {
// 2. find the index of the minimum frame number,
auto min_frame_idx = std::distance(
frame_numbers.begin(),
std::min_element(frame_numbers.begin(), frame_numbers.end()));
// 3. increase its index and update its respective frame number
frame_indices[min_frame_idx]++;
// 4. if we can't increase its index => throw error
if (frame_indices[min_frame_idx] >= total_frames()) {
throw std::runtime_error(LOCATION +
"Frame number out of range");
}
auto subfile_id =
frame_indices[min_frame_idx] / m_master.max_frames_per_file();
frame_numbers[min_frame_idx] =
subfiles[subfile_id][min_frame_idx]->frame_number(
frame_indices[min_frame_idx] %
m_master.max_frames_per_file());
}
}
if (m_master.geometry().col == 1) {
// get the part from each subfile and copy it to the frame
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
auto corrected_idx = frame_indices[part_idx];
auto subfile_id = corrected_idx / m_master.max_frames_per_file();
auto part_offset = subfiles[subfile_id][part_idx]->bytes_per_part();
subfiles[subfile_id][part_idx]->get_part(
frame_buffer + part_idx * part_offset,
corrected_idx % m_master.max_frames_per_file());
}
} else {
// create a buffer that will hold a the frame part
auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() *
m_master.bitdepth() /
8; // TODO! replace with image_size_in_bytes
auto *part_buffer = new std::byte[bytes_per_part];
// TODO! if we have many submodules we should reorder them on the module
// level
for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
auto corrected_idx = frame_indices[part_idx];
auto subfile_id = corrected_idx / m_master.max_frames_per_file();
subfiles[subfile_id][part_idx]->get_part(
part_buffer, corrected_idx % m_master.max_frames_per_file());
for (size_t cur_row = 0; cur_row < (m_master.pixels_y());
cur_row++) {
auto irow = cur_row + (part_idx / m_master.geometry().col) *
m_master.pixels_y();
auto icol =
(part_idx % m_master.geometry().col) * m_master.pixels_x();
auto dest = (irow * this->m_cols + icol);
dest = dest * m_master.bitdepth() / 8;
memcpy(frame_buffer + dest,
part_buffer + cur_row * m_master.pixels_x() *
m_master.bitdepth() / 8,
m_master.pixels_x() * m_master.bitdepth() / 8);
}
}
delete[] part_buffer;
}
}
std::vector<Frame> RawFile::read_n(size_t n_frames) {
// TODO: implement this in a more efficient way
std::vector<Frame> frames;
for (size_t i = 0; i < n_frames; i++) {
frames.push_back(this->get_frame(m_current_frame));
m_current_frame++;
}
return frames;
}
size_t RawFile::frame_number(size_t frame_index) {
if (frame_index >= m_master.frames_in_file()) {
throw std::runtime_error(LOCATION + " Frame number out of range");
}
size_t subfile_id = frame_index / m_master.max_frames_per_file();
if (subfile_id >= subfiles.size()) {
throw std::runtime_error(
LOCATION + " Subfile out of range. Possible missing data.");
}
return subfiles[subfile_id][0]->frame_number(
frame_index % m_master.max_frames_per_file());
}
RawFile::~RawFile() {
// TODO! Fix this, for file closing
for (auto &vec : subfiles) {
for (auto *subfile : vec) {
delete subfile;
}
}
}
} // namespace aare

150
src/RawFile.test.cpp Normal file
View File

@ -0,0 +1,150 @@
#include "aare/File.hpp"
#include <catch2/catch_test_macros.hpp>
#include <filesystem>
#include "test_config.hpp"
using aare::File;
TEST_CASE("Read number of frames from a jungfrau raw file") {
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
File f(fpath, "r");
REQUIRE(f.total_frames() == 10);
}
TEST_CASE("Read frame numbers from a jungfrau raw file") {
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
File f(fpath, "r");
// we know this file has 10 frames with frame numbers 1 to 10
// f0 1,2,3
// f1 4,5,6
// f2 7,8,9
// f3 10
for (size_t i = 0; i < 10; i++) {
CHECK(f.frame_number(i) == i + 1);
}
}
TEST_CASE("Read a frame number too high throws") {
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
File f(fpath, "r");
// we know this file has 10 frames with frame numbers 1 to 10
// f0 1,2,3
// f1 4,5,6
// f2 7,8,9
// f3 10
REQUIRE_THROWS(f.frame_number(10));
}
TEST_CASE("Read a frame numbers where the subfile is missing throws") {
auto fpath = test_data_path() / "jungfrau" / "jungfrau_missing_subfile_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
File f(fpath, "r");
// we know this file has 10 frames with frame numbers 1 to 10
// f0 1,2,3
// f1 4,5,6 - but files f1-f3 are missing
// f2 7,8,9 - gone
// f3 10 - gone
REQUIRE(f.frame_number(0) == 1);
REQUIRE(f.frame_number(1) == 2);
REQUIRE(f.frame_number(2) == 3);
REQUIRE_THROWS(f.frame_number(4));
REQUIRE_THROWS(f.frame_number(7));
REQUIRE_THROWS(f.frame_number(937));
REQUIRE_THROWS(f.frame_number(10));
}
TEST_CASE("Read data from a jungfrau 500k single port raw file") {
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
File f(fpath, "r");
// we know this file has 10 frames with pixel 0,0 being: 2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102
std::vector<uint16_t> pixel_0_0 = {2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102};
for (size_t i = 0; i < 10; i++) {
auto frame = f.read_frame();
CHECK(frame.rows() == 512);
CHECK(frame.cols() == 1024);
CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
}
}
TEST_CASE("Read frame numbers from a raw file") {
auto fpath = test_data_path() / "eiger" / "eiger_500k_16bit_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
// we know this file has 3 frames with frame numbers 14, 15, 16
std::vector<size_t> frame_numbers = {14, 15, 16};
File f(fpath, "r");
for (size_t i = 0; i < 3; i++) {
CHECK(f.frame_number(i) == frame_numbers[i]);
}
}
TEST_CASE("Compare reading from a numpy file with a raw file") {
auto fpath_raw = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
REQUIRE(std::filesystem::exists(fpath_raw));
auto fpath_npy = test_data_path() / "jungfrau" / "jungfrau_single_0.npy";
REQUIRE(std::filesystem::exists(fpath_npy));
File raw(fpath_raw, "r");
File npy(fpath_npy, "r");
CHECK(raw.total_frames() == 10);
CHECK(npy.total_frames() == 10);
for (size_t i = 0; i < 10; ++i) {
auto raw_frame = raw.read_frame();
auto npy_frame = npy.read_frame();
CHECK((raw_frame.view<uint16_t>() == npy_frame.view<uint16_t>()));
}
}
TEST_CASE("Read multipart files") {
auto fpath = test_data_path() / "jungfrau" / "jungfrau_double_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
File f(fpath, "r");
// we know this file has 10 frames check read_multiport.py for the values
std::vector<uint16_t> pixel_0_0 = {2099, 2121, 2108, 2084, 2084, 2118, 2066, 2108, 2112, 2116};
std::vector<uint16_t> pixel_0_1 = {2842, 2796, 2865, 2798, 2805, 2817, 2852, 2789, 2792, 2833};
std::vector<uint16_t> pixel_255_1023 = {2149, 2037, 2115, 2102, 2118, 2090, 2036, 2071, 2073, 2142};
std::vector<uint16_t> pixel_511_1023 = {3231, 3169, 3167, 3162, 3168, 3160, 3171, 3171, 3169, 3171};
std::vector<uint16_t> pixel_1_0 = {2748, 2614, 2665, 2629, 2618, 2630, 2631, 2634, 2577, 2598};
for (size_t i = 0; i < 10; i++) {
auto frame = f.read_frame();
CHECK(frame.rows() == 512);
CHECK(frame.cols() == 1024);
CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
CHECK(frame.view<uint16_t>()(0, 1) == pixel_0_1[i]);
CHECK(frame.view<uint16_t>()(1, 0) == pixel_1_0[i]);
CHECK(frame.view<uint16_t>()(255, 1023) == pixel_255_1023[i]);
CHECK(frame.view<uint16_t>()(511, 1023) == pixel_511_1023[i]);
}
}
TEST_CASE("Read file with unordered frames") {
//TODO! Better explanation and error message
auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw";
REQUIRE(std::filesystem::exists(fpath));
File f(fpath);
REQUIRE_THROWS((f.read_frame()));
}

328
src/RawMasterFile.cpp Normal file
View File

@ -0,0 +1,328 @@
#include "aare/RawMasterFile.hpp"
#include <sstream>
namespace aare {
RawFileNameComponents::RawFileNameComponents(
const std::filesystem::path &fname) {
m_base_path = fname.parent_path();
m_base_name = fname.stem();
m_ext = fname.extension();
if (m_ext != ".json" && m_ext != ".raw") {
throw std::runtime_error(LOCATION +
"Unsupported file type. (only .json or .raw)");
}
// parse file index
try {
auto pos = m_base_name.rfind('_');
m_file_index = std::stoi(m_base_name.substr(pos + 1));
} catch (const std::invalid_argument &e) {
throw std::runtime_error(LOCATION + "Could not parse file index");
}
// remove master from base name
auto pos = m_base_name.find("_master_");
if (pos != std::string::npos) {
m_base_name.erase(pos);
} else {
throw std::runtime_error(LOCATION +
"Could not find _master_ in file name");
}
}
std::filesystem::path RawFileNameComponents::master_fname() const {
return m_base_path /
fmt::format("{}_master_{}{}", m_base_name, m_file_index, m_ext);
}
std::filesystem::path RawFileNameComponents::data_fname(size_t mod_id,
size_t file_id) const {
return m_base_path / fmt::format("{}_d{}_f{}_{}.raw", m_base_name, mod_id,
file_id, m_file_index);
}
const std::filesystem::path &RawFileNameComponents::base_path() const {
return m_base_path;
}
const std::string &RawFileNameComponents::base_name() const {
return m_base_name;
}
const std::string &RawFileNameComponents::ext() const { return m_ext; }
int RawFileNameComponents::file_index() const { return m_file_index; }
// "[enabled\ndac dac 4\nstart 500\nstop 2200\nstep 5\nsettleTime 100us\n]"
ScanParameters::ScanParameters(const std::string& par){
std::istringstream iss(par.substr(1, par.size()-2));
std::string line;
while(std::getline(iss, line)){
if(line == "enabled"){
m_enabled = true;
}else if(line.find("dac") != std::string::npos){
m_dac = line.substr(4);
}else if(line.find("start") != std::string::npos){
m_start = std::stoi(line.substr(6));
}else if(line.find("stop") != std::string::npos){
m_stop = std::stoi(line.substr(5));
}else if(line.find("step") != std::string::npos){
m_step = std::stoi(line.substr(5));
}
}
}
int ScanParameters::start() const { return m_start; }
int ScanParameters::stop() const { return m_stop; }
int ScanParameters::step() const { return m_step; }
const std::string &ScanParameters::dac() const { return m_dac; }
bool ScanParameters::enabled() const { return m_enabled; }
RawMasterFile::RawMasterFile(const std::filesystem::path &fpath)
: m_fnc(fpath) {
if (!std::filesystem::exists(fpath)) {
throw std::runtime_error(LOCATION + " File does not exist");
}
if (m_fnc.ext() == ".json") {
parse_json(fpath);
} else if (m_fnc.ext() == ".raw") {
parse_raw(fpath);
} else {
throw std::runtime_error(LOCATION + "Unsupported file type");
}
}
std::filesystem::path RawMasterFile::data_fname(size_t mod_id,
size_t file_id) const {
return m_fnc.data_fname(mod_id, file_id);
}
const std::string &RawMasterFile::version() const { return m_version; }
const DetectorType &RawMasterFile::detector_type() const { return m_type; }
const TimingMode &RawMasterFile::timing_mode() const { return m_timing_mode; }
size_t RawMasterFile::image_size_in_bytes() const {
return m_image_size_in_bytes;
}
size_t RawMasterFile::frames_in_file() const { return m_frames_in_file; }
size_t RawMasterFile::pixels_y() const { return m_pixels_y; }
size_t RawMasterFile::pixels_x() const { return m_pixels_x; }
size_t RawMasterFile::max_frames_per_file() const {
return m_max_frames_per_file;
}
size_t RawMasterFile::bitdepth() const { return m_bitdepth; }
size_t RawMasterFile::frame_padding() const { return m_frame_padding; }
const FrameDiscardPolicy &RawMasterFile::frame_discard_policy() const {
return m_frame_discard_policy;
}
size_t RawMasterFile::total_frames_expected() const {
return m_total_frames_expected;
}
std::optional<size_t> RawMasterFile::number_of_rows() const {
return m_number_of_rows;
}
xy RawMasterFile::geometry() const { return m_geometry; }
std::optional<uint8_t> RawMasterFile::quad() const { return m_quad; }
// optional values, these may or may not be present in the master file
// and are therefore modeled as std::optional
std::optional<size_t> RawMasterFile::analog_samples() const {
return m_analog_samples;
}
std::optional<size_t> RawMasterFile::digital_samples() const {
return m_digital_samples;
}
std::optional<size_t> RawMasterFile::transceiver_samples() const {
return m_transceiver_samples;
}
ScanParameters RawMasterFile::scan_parameters() const {
return m_scan_parameters;
}
void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
std::ifstream ifs(fpath);
json j;
ifs >> j;
double v = j["Version"];
m_version = fmt::format("{:.1f}", v);
m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
m_timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
m_image_size_in_bytes = j["Image Size in bytes"];
m_frames_in_file = j["Frames in File"];
m_pixels_y = j["Pixels"]["y"];
m_pixels_x = j["Pixels"]["x"];
m_max_frames_per_file = j["Max Frames Per File"];
// Not all detectors write the bitdepth but in case
// its not there it is 16
try {
m_bitdepth = j.at("Dynamic Range");
} catch (const json::out_of_range &e) {
m_bitdepth = 16;
}
m_total_frames_expected = j["Total Frames"];
m_frame_padding = j["Frame Padding"];
m_frame_discard_policy = StringTo<FrameDiscardPolicy>(
j["Frame Discard Policy"].get<std::string>());
try {
m_number_of_rows = j.at("Number of rows");
} catch (const json::out_of_range &e) {
// keep the optional empty
}
// ----------------------------------------------------------------
// Special treatment of analog flag because of Moench03
try{
m_analog_flag = j.at("Analog Flag");
}catch (const json::out_of_range &e) {
// if it doesn't work still set it to one
// to try to decode analog samples (Old Moench03)
m_analog_flag = 1;
}
try {
if (m_analog_flag) {
m_analog_samples = j.at("Analog Samples");
}
} catch (const json::out_of_range &e) {
// keep the optional empty
// and set analog flag to 0
m_analog_flag = 0;
}
//-----------------------------------------------------------------
try {
m_quad = j.at("Quad");
} catch (const json::out_of_range &e) {
// keep the optional empty
}
// try{
// std::string adc_mask = j.at("ADC Mask");
// m_adc_mask = std::stoul(adc_mask, nullptr, 16);
// }catch (const json::out_of_range &e) {
// m_adc_mask = 0;
// }
try {
int digital_flag = j.at("Digital Flag");
if (digital_flag) {
m_digital_samples = j.at("Digital Samples");
}
} catch (const json::out_of_range &e) {
// keep the optional empty
}
try{
m_transceiver_flag = j.at("Transceiver Flag");
if(m_transceiver_flag){
m_transceiver_samples = j.at("Transceiver Samples");
}
}catch (const json::out_of_range &e) {
// keep the optional empty
}
try{
std::string scan_parameters = j.at("Scan Parameters");
m_scan_parameters = ScanParameters(scan_parameters);
}catch (const json::out_of_range &e) {
// not a scan
}
// Update detector type for Moench
// TODO! How does this work with old .raw master files?
#ifdef AARE_VERBOSE
fmt::print("Detecting Moench03: m_pixels_y: {}, m_analog_samples: {}\n",
m_pixels_y, m_analog_samples.value_or(0));
#endif
if (m_type == DetectorType::Moench && !m_analog_samples &&
m_pixels_y == 400) {
m_type = DetectorType::Moench03;
} else if (m_type == DetectorType::Moench && m_pixels_y == 400 &&
m_analog_samples == 5000) {
m_type = DetectorType::Moench03_old;
}
}
void RawMasterFile::parse_raw(const std::filesystem::path &fpath) {
std::ifstream ifs(fpath);
for (std::string line; std::getline(ifs, line);) {
if (line == "#Frame Header")
break;
auto pos = line.find(':');
auto key_pos = pos;
while (key_pos != std::string::npos && std::isspace(line[--key_pos]))
;
if (key_pos != std::string::npos) {
auto key = line.substr(0, key_pos + 1);
auto value = line.substr(pos + 2);
// do the actual parsing
if (key == "Version") {
m_version = value;
} else if (key == "TimeStamp") {
} else if (key == "Detector Type") {
m_type = StringTo<DetectorType>(value);
} else if (key == "Timing Mode") {
m_timing_mode = StringTo<TimingMode>(value);
} else if (key == "Image Size") {
m_image_size_in_bytes = std::stoi(value);
} else if (key == "Frame Padding") {
m_frame_padding = std::stoi(value);
// } else if (key == "Frame Discard Policy"){
// m_frame_discard_policy =
// StringTo<FrameDiscardPolicy>(value);
// } else if (key == "Number of rows"){
// m_number_of_rows = std::stoi(value);
} else if (key == "Analog Flag") {
m_analog_flag = std::stoi(value);
} else if (key == "Digital Flag") {
m_digital_flag = std::stoi(value);
} else if (key == "Analog Samples") {
if (m_analog_flag == 1) {
m_analog_samples = std::stoi(value);
}
} else if (key == "Digital Samples") {
if (m_digital_flag == 1) {
m_digital_samples = std::stoi(value);
}
} else if (key == "Frames in File") {
m_frames_in_file = std::stoi(value);
// } else if (key == "ADC Mask") {
// m_adc_mask = std::stoi(value, nullptr, 16);
} else if (key == "Pixels") {
// Total number of pixels cannot be found yet looking at
// submodule
pos = value.find(',');
m_pixels_x = std::stoi(value.substr(1, pos));
m_pixels_y = std::stoi(value.substr(pos + 1));
} else if (key == "Total Frames") {
m_total_frames_expected = std::stoi(value);
} else if (key == "Dynamic Range") {
m_bitdepth = std::stoi(value);
} else if (key == "Quad") {
m_quad = std::stoi(value);
} else if (key == "Max Frames Per File") {
m_max_frames_per_file = std::stoi(value);
} else if (key == "Geometry") {
pos = value.find(',');
m_geometry = {
static_cast<uint32_t>(std::stoi(value.substr(1, pos))),
static_cast<uint32_t>(std::stoi(value.substr(pos + 1)))};
}
}
}
}
} // namespace aare

278
src/RawMasterFile.test.cpp Normal file
View File

@ -0,0 +1,278 @@
#include "aare/RawMasterFile.hpp"
#include <catch2/catch_test_macros.hpp>
#include "test_config.hpp"
using namespace aare;
TEST_CASE("Parse a master file fname"){
RawFileNameComponents m("test_master_1.json");
REQUIRE(m.base_name() == "test");
REQUIRE(m.ext() == ".json");
REQUIRE(m.file_index() == 1);
REQUIRE(m.base_path() == "");
}
TEST_CASE("Extraction of base path works"){
RawFileNameComponents m("some/path/test_master_73.json");
REQUIRE(m.base_name() == "test");
REQUIRE(m.ext() == ".json");
REQUIRE(m.file_index() == 73);
REQUIRE(m.base_path() == "some/path");
}
TEST_CASE("Construction of master file name and data files"){
RawFileNameComponents m("test_master_1.json");
REQUIRE(m.master_fname() == "test_master_1.json");
REQUIRE(m.data_fname(0, 0) == "test_d0_f0_1.raw");
REQUIRE(m.data_fname(1, 0) == "test_d1_f0_1.raw");
REQUIRE(m.data_fname(0, 1) == "test_d0_f1_1.raw");
REQUIRE(m.data_fname(1, 1) == "test_d1_f1_1.raw");
}
TEST_CASE("Master file name does not fit pattern"){
REQUIRE_THROWS(RawFileNameComponents("somefile.json"));
REQUIRE_THROWS(RawFileNameComponents("another_test_d0_f0_1.raw"));
REQUIRE_THROWS(RawFileNameComponents("test_master_1.txt"));
}
TEST_CASE("Parse scan parameters"){
ScanParameters s("[enabled\ndac dac 4\nstart 500\nstop 2200\nstep 5\nsettleTime 100us\n]");
REQUIRE(s.enabled());
REQUIRE(s.dac() == "dac 4");
REQUIRE(s.start() == 500);
REQUIRE(s.stop() == 2200);
REQUIRE(s.step() == 5);
}
TEST_CASE("A disabled scan"){
ScanParameters s("[disabled]");
REQUIRE_FALSE(s.enabled());
REQUIRE(s.dac() == "");
REQUIRE(s.start() == 0);
REQUIRE(s.stop() == 0);
REQUIRE(s.step() == 0);
}
TEST_CASE("Parse a master file in .json format"){
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
RawMasterFile f(fpath);
// "Version": 7.2,
REQUIRE(f.version() == "7.2");
// "Timestamp": "Tue Feb 20 08:28:24 2024",
// "Detector Type": "Jungfrau",
REQUIRE(f.detector_type() == DetectorType::Jungfrau);
// "Timing Mode": "auto",
REQUIRE(f.timing_mode() == TimingMode::Auto);
// "Geometry": {
// "x": 1,
// "y": 1
// },
REQUIRE(f.geometry().col == 1);
REQUIRE(f.geometry().row == 1);
// "Image Size in bytes": 1048576,
REQUIRE(f.image_size_in_bytes() == 1048576);
// "Pixels": {
// "x": 1024,
REQUIRE(f.pixels_x() == 1024);
// "y": 512
REQUIRE(f.pixels_y() == 512);
// },
// "Max Frames Per File": 3,
REQUIRE(f.max_frames_per_file() == 3);
//Jungfrau doesn't write but it is 16
REQUIRE(f.bitdepth() == 16);
// "Frame Discard Policy": "nodiscard",
// "Frame Padding": 1,
REQUIRE(f.frame_padding() == 1);
// "Scan Parameters": "[disabled]",
// "Total Frames": 10,
REQUIRE(f.total_frames_expected() == 10);
// "Receiver Roi": {
// "xmin": 4294967295,
// "xmax": 4294967295,
// "ymin": 4294967295,
// "ymax": 4294967295
// },
// "Exptime": "10us",
// "Period": "1ms",
// "Number of UDP Interfaces": 1,
// "Number of rows": 512,
REQUIRE(f.number_of_rows() == 512);
// "Frames in File": 10,
REQUIRE(f.frames_in_file() == 10);
//TODO! Should we parse this?
// "Frame Header Format": {
// "Frame Number": "8 bytes",
// "SubFrame Number/ExpLength": "4 bytes",
// "Packet Number": "4 bytes",
// "Bunch ID": "8 bytes",
// "Timestamp": "8 bytes",
// "Module Id": "2 bytes",
// "Row": "2 bytes",
// "Column": "2 bytes",
// "Reserved": "2 bytes",
// "Debug": "4 bytes",
// "Round Robin Number": "2 bytes",
// "Detector Type": "1 byte",
// "Header Version": "1 byte",
// "Packets Caught Mask": "64 bytes"
// }
// }
REQUIRE_FALSE(f.analog_samples());
REQUIRE_FALSE(f.digital_samples());
}
TEST_CASE("Parse a master file in .raw format"){
auto fpath = test_data_path() / "moench/moench04_noise_200V_sto_both_100us_no_light_thresh_900_master_0.raw";
REQUIRE(std::filesystem::exists(fpath));
RawMasterFile f(fpath);
// Version : 6.4
REQUIRE(f.version() == "6.4");
// TimeStamp : Wed Aug 31 09:08:49 2022
// Detector Type : ChipTestBoard
REQUIRE(f.detector_type() == DetectorType::ChipTestBoard);
// Timing Mode : auto
REQUIRE(f.timing_mode() == TimingMode::Auto);
// Geometry : [1, 1]
REQUIRE(f.geometry().col == 1);
REQUIRE(f.geometry().row == 1);
// Image Size : 360000 bytes
REQUIRE(f.image_size_in_bytes() == 360000);
// Pixels : [96, 1]
REQUIRE(f.pixels_x() == 96);
REQUIRE(f.pixels_y() == 1);
// Max Frames Per File : 20000
REQUIRE(f.max_frames_per_file() == 20000);
// Frame Discard Policy : nodiscard
// Frame Padding : 1
REQUIRE(f.frame_padding() == 1);
// Scan Parameters : [disabled]
// Total Frames : 100
REQUIRE(f.total_frames_expected() == 100);
// Exptime : 100us
// Period : 4ms
// Ten Giga : 1
// ADC Mask : 0xffffffff
// Analog Flag : 1
// Analog Samples : 5000
REQUIRE(f.analog_samples() == 5000);
// Digital Flag : 1
// Digital Samples : 5000
REQUIRE(f.digital_samples() == 5000);
// Dbit Offset : 0
// Dbit Bitset : 0
// Frames in File : 100
REQUIRE(f.frames_in_file() == 100);
// #Frame Header
// Frame Number : 8 bytes
// SubFrame Number/ExpLength : 4 bytes
// Packet Number : 4 bytes
// Bunch ID : 8 bytes
// Timestamp : 8 bytes
// Module Id : 2 bytes
// Row : 2 bytes
// Column : 2 bytes
// Reserved : 2 bytes
// Debug : 4 bytes
// Round Robin Number : 2 bytes
// Detector Type : 1 byte
// Header Version : 1 byte
// Packets Caught Mask : 64 bytes
}
TEST_CASE("Read eiger master file"){
auto fpath = test_data_path() / "eiger" / "eiger_500k_32bit_master_0.json";
REQUIRE(std::filesystem::exists(fpath));
RawMasterFile f(fpath);
// {
// "Version": 7.2,
REQUIRE(f.version() == "7.2");
// "Timestamp": "Tue Mar 26 17:24:34 2024",
// "Detector Type": "Eiger",
REQUIRE(f.detector_type() == DetectorType::Eiger);
// "Timing Mode": "auto",
REQUIRE(f.timing_mode() == TimingMode::Auto);
// "Geometry": {
// "x": 2,
// "y": 2
// },
// "Image Size in bytes": 524288,
REQUIRE(f.image_size_in_bytes() == 524288);
// "Pixels": {
// "x": 512,
REQUIRE(f.pixels_x() == 512);
// "y": 256
REQUIRE(f.pixels_y() == 256);
// },
// "Max Frames Per File": 10000,
REQUIRE(f.max_frames_per_file() == 10000);
// "Frame Discard Policy": "nodiscard",
REQUIRE(f.frame_discard_policy() == FrameDiscardPolicy::NoDiscard);
// "Frame Padding": 1,
REQUIRE(f.frame_padding() == 1);
// "Scan Parameters": "[disabled]",
// "Total Frames": 3,
// "Receiver Roi": {
// "xmin": 4294967295,
// "xmax": 4294967295,
// "ymin": 4294967295,
// "ymax": 4294967295
// },
// "Dynamic Range": 32,
// "Ten Giga": 0,
// "Exptime": "5s",
// "Period": "1s",
// "Threshold Energy": -1,
// "Sub Exptime": "2.62144ms",
// "Sub Period": "2.62144ms",
// "Quad": 0,
// "Number of rows": 256,
// "Rate Corrections": "[0, 0]",
// "Frames in File": 3,
// "Frame Header Format": {
// "Frame Number": "8 bytes",
// "SubFrame Number/ExpLength": "4 bytes",
// "Packet Number": "4 bytes",
// "Bunch ID": "8 bytes",
// "Timestamp": "8 bytes",
// "Module Id": "2 bytes",
// "Row": "2 bytes",
// "Column": "2 bytes",
// "Reserved": "2 bytes",
// "Debug": "4 bytes",
// "Round Robin Number": "2 bytes",
// "Detector Type": "1 byte",
// "Header Version": "1 byte",
// "Packets Caught Mask": "64 bytes"
// }
// }
}

91
src/SubFile.cpp Normal file
View File

@ -0,0 +1,91 @@
#include "aare/SubFile.hpp"
#include "aare/PixelMap.hpp"
#include <cstring> // memcpy
#include <fmt/core.h>
#include <iostream>
namespace aare {
SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
const std::string &mode)
: m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode), m_detector_type(detector) {
if (m_detector_type == DetectorType::Moench03_old) {
pixel_map = GenerateMoench03PixelMap();
}
if (std::filesystem::exists(fname)) {
n_frames = std::filesystem::file_size(fname) / (sizeof(DetectorHeader) + rows * cols * bitdepth / 8);
} else {
n_frames = 0;
}
if (mode == "r") {
fp = fopen(m_fname.string().c_str(), "rb");
} else {
throw std::runtime_error(LOCATION + "Unsupported mode. Can only read RawFiles.");
}
if (fp == nullptr) {
throw std::runtime_error(LOCATION + fmt::format("Could not open file {}", m_fname.string()));
}
#ifdef AARE_VERBOSE
fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames);
fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols, m_bitdepth);
#endif
}
size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {
if (frame_index >= n_frames) {
throw std::runtime_error("Frame number out of range");
}
fseek(fp, sizeof(DetectorHeader) + (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, // NOLINT
SEEK_SET);
if (pixel_map){
// read into a temporary buffer and then copy the data to the buffer
// in the correct order
auto part_buffer = new std::byte[bytes_per_part()];
auto wc = fread(part_buffer, bytes_per_part(), 1, fp);
auto *data = reinterpret_cast<uint16_t *>(buffer);
auto *part_data = reinterpret_cast<uint16_t *>(part_buffer);
for (size_t i = 0; i < pixels_per_part(); i++) {
data[i] = part_data[(*pixel_map)(i)];
}
delete[] part_buffer;
return wc;
}else{
// read directly into the buffer
return fread(buffer, this->bytes_per_part(), 1, this->fp);
}
}
size_t SubFile::write_part(std::byte *buffer, DetectorHeader header, size_t frame_index) {
if (frame_index > n_frames) {
throw std::runtime_error("Frame number out of range");
}
fseek(fp, static_cast<int64_t>((sizeof(DetectorHeader) + bytes_per_part()) * frame_index), SEEK_SET);
auto wc = fwrite(reinterpret_cast<char *>(&header), sizeof(header), 1, fp);
wc += fwrite(buffer, bytes_per_part(), 1, fp);
return wc;
}
size_t SubFile::frame_number(size_t frame_index) {
DetectorHeader h{};
fseek(fp, (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
if (rc != 1)
throw std::runtime_error(LOCATION + "Could not read header from file");
return h.frameNumber;
}
SubFile::~SubFile() {
if (fp) {
fclose(fp);
}
}
} // namespace aare

83
src/defs.cpp Normal file
View File

@ -0,0 +1,83 @@
#include "aare/defs.hpp"
#include <stdexcept>
#include <string>
namespace aare {
/**
* @brief Convert a DetectorType to a string
* @param type DetectorType
* @return string representation of the DetectorType
*/
template <> std::string ToString(DetectorType arg) {
switch (arg) {
case DetectorType::Jungfrau:
return "Jungfrau";
case DetectorType::Eiger:
return "Eiger";
case DetectorType::Mythen3:
return "Mythen3";
case DetectorType::Moench:
return "Moench";
case DetectorType::Moench03:
return "Moench03";
case DetectorType::Moench03_old:
return "Moench03_old";
case DetectorType::ChipTestBoard:
return "ChipTestBoard";
default:
return "Unknown";
}
}
/**
* @brief Convert a string to a DetectorType
* @param name string representation of the DetectorType
* @return DetectorType
* @throw runtime_error if the string does not match any DetectorType
*/
template <> DetectorType StringTo(const std::string &arg) {
if (arg == "Jungfrau")
return DetectorType::Jungfrau;
if (arg == "Eiger")
return DetectorType::Eiger;
if (arg == "Mythen3")
return DetectorType::Mythen3;
if (arg == "Moench")
return DetectorType::Moench;
if (arg == "Moench03")
return DetectorType::Moench03;
if (arg == "Moench03_old")
return DetectorType::Moench03_old;
if (arg == "ChipTestBoard")
return DetectorType::ChipTestBoard;
throw std::runtime_error("Could not decode dector from: \"" + arg + "\"");
}
/**
* @brief Convert a string to a TimingMode
* @param mode string representation of the TimingMode
* @return TimingMode
* @throw runtime_error if the string does not match any TimingMode
*/
template <> TimingMode StringTo(const std::string &arg) {
if (arg == "auto")
return TimingMode::Auto;
if (arg == "trigger")
return TimingMode::Trigger;
throw std::runtime_error("Could not decode timing mode from: \"" + arg + "\"");
}
template <> FrameDiscardPolicy StringTo(const std::string &arg) {
if (arg == "nodiscard")
return FrameDiscardPolicy::NoDiscard;
if (arg == "discard")
return FrameDiscardPolicy::Discard;
if (arg == "discardpartial")
return FrameDiscardPolicy::DiscardPartial;
throw std::runtime_error("Could not decode frame discard policy from: \"" + arg + "\"");
}
// template <> TimingMode StringTo<TimingMode>(std::string mode);
} // namespace aare

42
src/defs.test.cpp Normal file
View File

@ -0,0 +1,42 @@
#include "aare/defs.hpp"
// #include "aare/utils/floats.hpp"
#include <catch2/catch_test_macros.hpp>
#include <string>
TEST_CASE("Enum to string conversion") {
// By the way I don't think the enum string conversions should be in the defs.hpp file
// but let's use this to show a test
REQUIRE(ToString(aare::DetectorType::Jungfrau) == "Jungfrau");
}
TEST_CASE("Cluster creation") {
aare::Cluster c(13, 15);
REQUIRE(c.cluster_sizeX == 13);
REQUIRE(c.cluster_sizeY == 15);
REQUIRE(c.dt == aare::Dtype(typeid(int32_t)));
REQUIRE(c.data() != nullptr);
aare::Cluster c2(c);
REQUIRE(c2.cluster_sizeX == 13);
REQUIRE(c2.cluster_sizeY == 15);
REQUIRE(c2.dt == aare::Dtype(typeid(int32_t)));
REQUIRE(c2.data() != nullptr);
}
// TEST_CASE("cluster set and get data") {
// aare::Cluster c2(33, 44, aare::Dtype(typeid(double)));
// REQUIRE(c2.cluster_sizeX == 33);
// REQUIRE(c2.cluster_sizeY == 44);
// REQUIRE(c2.dt == aare::Dtype::DOUBLE);
// double v = 3.14;
// c2.set<double>(0, v);
// double v2 = c2.get<double>(0);
// REQUIRE(aare::compare_floats<double>(v, v2));
// c2.set<double>(33 * 44 - 1, 123.11);
// double v3 = c2.get<double>(33 * 44 - 1);
// REQUIRE(aare::compare_floats<double>(123.11, v3));
// REQUIRE_THROWS_AS(c2.set(0, 1), std::invalid_argument); // set int to double
// }

44
tests/CMakeLists.txt Normal file
View File

@ -0,0 +1,44 @@
# Download catch2 if configured to do so
if (AARE_FETCH_CATCH)
FetchContent_Declare(
Catch2
GIT_SHALLOW TRUE
GIT_REPOSITORY https://github.com/catchorg/Catch2.git
GIT_TAG v3.5.3
)
FetchContent_MakeAvailable(Catch2)
else()
# Otherwise look for installed catch2
find_package(Catch2 3 REQUIRED)
endif()
list(APPEND CMAKE_MODULE_PATH ${Catch2_SOURCE_DIR}/extras)
add_executable(tests test.cpp)
target_link_libraries(tests PRIVATE Catch2::Catch2WithMain)
set_target_properties(tests PROPERTIES
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
OUTPUT_NAME run_tests
)
include(CTest)
include(Catch)
catch_discover_tests(tests)
set(TestSources
${CMAKE_CURRENT_SOURCE_DIR}/test.cpp
)
target_sources(tests PRIVATE ${TestSources} )
#Work around to remove, this is not the way to do it =)
target_link_libraries(tests PRIVATE aare_core aare_compiler_flags)
#configure a header to pass test file paths
get_filename_component(TEST_FILE_PATH ${PROJECT_SOURCE_DIR}/data ABSOLUTE)
configure_file(test_config.hpp.in test_config.hpp)
target_include_directories(tests PRIVATE ${CMAKE_CURRENT_BINARY_DIR})

21
tests/test.cpp Normal file
View File

@ -0,0 +1,21 @@
#include "test_config.hpp"
#include <catch2/catch_test_macros.hpp>
#include <climits>
#include <filesystem>
#include <fstream>
TEST_CASE("Test suite can find data assets") {
auto fpath = test_data_path() / "numpy" / "test_numpy_file.npy";
REQUIRE(std::filesystem::exists(fpath));
}
TEST_CASE("Test suite can open data assets") {
auto fpath = test_data_path() / "numpy" / "test_numpy_file.npy";
auto f = std::ifstream(fpath, std::ios::binary);
REQUIRE(f.is_open());
}
TEST_CASE("Test float32 and char8") {
REQUIRE(sizeof(float) == 4);
REQUIRE(CHAR_BIT == 8);
}

12
tests/test_config.hpp.in Normal file
View File

@ -0,0 +1,12 @@
#pragma once
#include <filesystem>
#include <cstdlib>
inline auto test_data_path(){
if(const char* env_p = std::getenv("AARE_TEST_DATA")){
return std::filesystem::path(env_p);
}else{
throw std::runtime_error("AARE_TEST_DATA_PATH not set");
}
}