fixed linking issue in arping and shm test (#659)

Co-authored-by: Erik Frojdh <erik.frojdh@psi.ch>

.clang-tidy

@@ -19,6 +19,7 @@ Checks:          '*,
                     -google-readability-braces-around-statements,
                     -modernize-use-trailing-return-type,
                     -readability-isolate-declaration,
+                    -readability-implicit-bool-conversion,
                     -llvmlibc-*'
 
 HeaderFilterRegex: \.h

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "python/pybind11"]
-	path = libs/pybind11
-	url = https://github.com/pybind/pybind11.git

CMakeLists.txt

@@ -1,13 +1,15 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 cmake_minimum_required(VERSION 3.12)
 project(slsDetectorPackage)
-set(PROJECT_VERSION 5.1.0)
-include(CheckIPOSupported)
+set(PROJECT_VERSION 7.0.0)
+
 set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
 
 cmake_policy(SET CMP0074 NEW)
 include(cmake/project_version.cmake)
-
-# Include additional modules that are used unconditionally
+include(cmake/SlsAddFlag.cmake)
+include(cmake/SlsFindZeroMQ.cmake)
 include(GNUInstallDirs)
 
 # If conda build, always set lib dir to 'lib'
@@ -21,7 +23,7 @@ string(TOLOWER "${PROJECT_NAME}" PROJECT_NAME_LOWER)
 
 # Set targets export name (used by slsDetectorPackage and dependencies)
 set(TARGETS_EXPORT_NAME "${PROJECT_NAME_LOWER}-targets")
-#set(namespace "${PROJECT_NAME}::")
+set(namespace "sls::")
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
 
@@ -32,6 +34,8 @@ if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
     set(SLS_MASTER_PROJECT ON)
 endif()
 
+
+
 option(SLS_USE_HDF5 "HDF5 File format" OFF)
 option(SLS_BUILD_SHARED_LIBRARIES "Build shared libaries" ON)
 option(SLS_USE_TEXTCLIENT "Text Client" ON)
@@ -44,6 +48,7 @@ option(SLS_USE_TESTS "TESTS" OFF)
 option(SLS_USE_INTEGRATION_TESTS "Integration Tests" OFF)
 option(SLS_USE_SANITIZER "Sanitizers for debugging" OFF)
 option(SLS_USE_PYTHON "Python bindings" OFF)
+option(SLS_INSTALL_PYTHONEXT "Install the python extension in the install tree under CMAKE_INSTALL_PREFIX/python/" OFF)
 option(SLS_USE_CTBGUI "ctb GUI" OFF)
 option(SLS_BUILD_DOCS "docs" OFF)
 option(SLS_BUILD_EXAMPLES "examples" OFF)
@@ -51,7 +56,35 @@ option(SLS_TUNE_LOCAL "tune to local machine" OFF)
 option(SLS_DEVEL_HEADERS "install headers for devel" OFF)
 option(SLS_USE_MOENCH "compile zmq and post processing for Moench" OFF)
 
-# set(ClangFormat_BIN_NAME clang-format)
+#Convenience option to switch off defaults when building Moench binaries only
+option(SLS_BUILD_ONLY_MOENCH "compile only Moench" OFF)
+if(SLS_BUILD_ONLY_MOENCH)
+    message(STATUS "Build MOENCH binaries only!")
+    set(SLS_BUILD_SHARED_LIBRARIES OFF CACHE BOOL "Disabled for MOENCH_ONLY" FORCE)
+    set(SLS_USE_TEXTCLIENT OFF CACHE BOOL "Disabled for MOENCH_ONLY" FORCE)
+    set(SLS_USE_DETECTOR OFF CACHE BOOL "Disabled for MOENCH_ONLY" FORCE)
+    set(SLS_USE_RECEIVER OFF CACHE BOOL "Disabled for MOENCH_ONLY" FORCE)
+    set(SLS_USE_RECEIVER_BINARIES OFF CACHE BOOL "Disabled for MOENCH_ONLY" FORCE)
+    set(SLS_USE_MOENCH ON CACHE BOOL "Enable" FORCE)
+endif()
+
+
+option(SLS_EXT_BUILD "external build of part of the project" OFF)
+if(SLS_EXT_BUILD)
+    message(STATUS "External build using already installed libraries")
+    set(SLS_BUILD_SHARED_LIBRARIES OFF CACHE BOOL "Should already exist" FORCE)
+    set(SLS_USE_TEXTCLIENT OFF CACHE BOOL "Should already exist" FORCE)
+    set(SLS_USE_DETECTOR OFF CACHE BOOL "Should already exist" FORCE)
+    set(SLS_USE_RECEIVER OFF CACHE BOOL "Should already exist" FORCE)
+    set(SLS_USE_RECEIVER_BINARIES OFF CACHE BOOL "Should already exist" FORCE)
+    set(SLS_MASTER_PROJECT OFF CACHE BOOL "No master proj in case of extbuild" FORCE) 
+endif()
+
+#Maybe have an option guarding this?
+set(SLS_INTERNAL_RAPIDJSON_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libs/rapidjson)
+
+set(SLS_INTERNAL_QWT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libs/qwt-6.1.5)
+
 set(ClangFormat_EXCLUDE_PATTERNS    "build/" 
                                     "libs/" 
                                     "slsDetectorCalibration/" 
@@ -62,11 +95,6 @@ set(ClangFormat_EXCLUDE_PATTERNS    "build/"
                                     ${CMAKE_BINARY_DIR})
 find_package(ClangFormat)
 
-#Enable LTO if available
-check_ipo_supported(RESULT SLS_LTO_AVAILABLE)
-message(STATUS "SLS_LTO_AVAILABLE:" ${SLS_LTO_AVAILABLE})
-
-
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
@@ -75,70 +103,77 @@ if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 
-#Add two fake libraries to manage options
-add_library(slsProjectOptions INTERFACE)
-add_library(slsProjectWarnings INTERFACE)
-target_compile_features(slsProjectOptions INTERFACE cxx_std_11)
-target_compile_options(slsProjectWarnings INTERFACE 
-                                            -Wall
-                                            -Wextra
-                                            -Wno-unused-parameter #Needs to be slowly mitigated
-                                            # -Wold-style-cast
-                                            -Wnon-virtual-dtor
-                                            -Woverloaded-virtual
-                                            -Wdouble-promotion
-                                            -Wformat=2
-                                            -Wredundant-decls
-                                            # -Wconversion
-                                            -Wvla
-                                            -Wdouble-promotion
-                                            -Werror=return-type
-
-                                 )
-
-#Settings for C code
-add_library(slsProjectCSettings INTERFACE)
-target_compile_features(slsProjectCSettings INTERFACE c_std_99)
-target_compile_options(slsProjectCSettings INTERFACE 
-                                            -Wall
-                                            -Wextra
-                                            -Wno-unused-parameter
-                                            -Wdouble-promotion
-                                            -Wformat=2
-                                            -Wredundant-decls
-                                            -Wdouble-promotion
-                                            -Werror=return-type
-                                )
+#Enable LTO if available
+include(CheckIPOSupported)
+check_ipo_supported(RESULT SLS_LTO_AVAILABLE)
+if((CMAKE_BUILD_TYPE STREQUAL "Release") AND SLS_LTO_AVAILABLE)
+    message(STATUS "Building with link time optimization")
+else()
+    message(STATUS "Building without link time optimization")
+endif()
 
 
-#Testing for minimum version for compilers
-if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.2)
-        message(FATAL_ERROR "Clang version must be at least 3.2!")
-    endif()
-    target_compile_options(slsProjectWarnings INTERFACE -Wshadow) #Clag does not warn on constructor
-elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8)
-            message(FATAL_ERROR "GCC version must be at least 4.8!")
-    endif()
+if(SLS_EXT_BUILD)
+    # Find ourself in case of external build
+    find_package(slsDetectorPackage ${PROJECT_VERSION} REQUIRED)
+endif()
 
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5)
+
+
+# slsProjectOptions and slsProjectWarnings are used
+# to control options for the libraries
+if(NOT TARGET slsProjectOptions)
+    add_library(slsProjectOptions INTERFACE)
+    target_compile_features(slsProjectOptions INTERFACE cxx_std_11)
+endif()
+
+if (NOT TARGET slsProjectWarnings)
+    add_library(slsProjectWarnings INTERFACE)
+    target_compile_options(slsProjectWarnings INTERFACE 
+        -Wall
+        -Wextra
+        -Wno-unused-parameter
+        # -Wold-style-cast
+        -Wnon-virtual-dtor
+        -Woverloaded-virtual
+        -Wdouble-promotion
+        -Wformat=2
+        -Wredundant-decls
+        # -Wconversion
+        -Wvla
+        -Wdouble-promotion
+        -Werror=return-type
+    )
+    # Add or disable warnings depending on if the compiler supports them 
+    # The function checks internally and sets HAS_warning-name
+    sls_enable_cxx_warning("-Wnull-dereference")
+    sls_enable_cxx_warning("-Wduplicated-cond")
+    sls_disable_cxx_warning("-Wclass-memaccess")
+
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5 AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
             target_compile_options(slsProjectWarnings INTERFACE 
-                                             -Wno-missing-field-initializers)
+                                                -Wno-missing-field-initializers)
     endif()
-    
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 6.0)
-        target_compile_options(slsProjectWarnings INTERFACE 
-                                                -Wno-misleading-indentation # mostly in rapidjson remove using clang format
-                                                -Wduplicated-cond
-                                                -Wnull-dereference )
 
-    endif()
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.0)
-        target_compile_options(slsProjectWarnings INTERFACE 
-                                                -Wno-class-memaccess )
 
-    endif()
+endif()
+
+
+if (NOT TARGET slsProjectCSettings)
+    #Settings for C code
+    add_library(slsProjectCSettings INTERFACE)
+    target_compile_options(slsProjectCSettings INTERFACE 
+                                                -std=gnu99 #fixed
+                                                -Wall
+                                                -Wextra
+                                                -Wno-unused-parameter
+                                                -Wdouble-promotion
+                                                -Wformat=2
+                                                -Wredundant-decls
+                                                -Wdouble-promotion
+                                                -Werror=return-type
+                                    )
+    sls_disable_c_warning("-Wstringop-truncation")
 endif()
 
 
@@ -154,68 +189,32 @@ if(SLS_TUNE_LOCAL)
 endif()
 
 
-#rapidjson
-add_library(rapidjson INTERFACE)
-target_include_directories(rapidjson INTERFACE
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/libs/rapidjson>  
-)
-
-# Install fake the libraries
-install(TARGETS slsProjectOptions slsProjectWarnings rapidjson
+if(SLS_MASTER_PROJECT)
+install(TARGETS slsProjectOptions slsProjectWarnings 
     EXPORT "${TARGETS_EXPORT_NAME}"
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
     ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
     PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
 )
+endif()
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(CMAKE_INSTALL_RPATH $ORIGIN)
-# set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
 set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
 
 
-set(ZeroMQ_HINT "" CACHE STRING "Hint where ZeroMQ could be found")
-#Adapted from: https://github.com/zeromq/cppzmq/
-if (NOT TARGET libzmq)
-    if(ZeroMQ_HINT)
-        message(STATUS "Looking for ZeroMQ in: ${ZeroMQ_HINT}")
-        find_package(ZeroMQ 4 
-            NO_DEFAULT_PATH
-            HINTS ${ZeroMQ_DIR}
-        )
-    else()
-        find_package(ZeroMQ 4 QUIET)
-    endif()
-    
+custom_find_zmq()
 
 
-  # libzmq autotools install: fallback to pkg-config
-  if(NOT ZeroMQ_FOUND)
-    message(STATUS "CMake libzmq package not found, trying again with pkg-config (normal install of zeromq)")
-    list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/libzmq-pkg-config)
-    find_package(ZeroMQ 4 REQUIRED)
-  endif()
-
-  # TODO "REQUIRED" above should already cause a fatal failure if not found, but this doesn't seem to work
-  if(NOT ZeroMQ_FOUND)
-    message(FATAL_ERROR "ZeroMQ was not found, neither as a CMake package nor via pkg-config")
-  endif()
-
-  if (ZeroMQ_FOUND AND NOT TARGET libzmq)
-    message(FATAL_ERROR "ZeroMQ version not supported!")
-  endif()
-endif()
-
 if (SLS_USE_TESTS)
     enable_testing()
     add_subdirectory(tests)
 endif(SLS_USE_TESTS)
 
 
-
-
-# Common functionallity to detector and receiver
-add_subdirectory(slsSupportLib)
+if(NOT SLS_EXT_BUILD)
+    add_subdirectory(slsSupportLib)
+endif()
 
 if (SLS_USE_DETECTOR OR SLS_USE_TEXTCLIENT)
     add_subdirectory(slsDetectorSoftware)
@@ -226,6 +225,7 @@ if (SLS_USE_RECEIVER)
 endif (SLS_USE_RECEIVER)
 
 if (SLS_USE_GUI)   
+    add_subdirectory(libs/qwt)  
     add_subdirectory(slsDetectorGui)
 endif (SLS_USE_GUI)
 
@@ -239,7 +239,7 @@ endif (SLS_USE_INTEGRATION_TESTS)
 
 if (SLS_USE_PYTHON)
     find_package (Python 3.6 COMPONENTS Interpreter Development)
-    add_subdirectory(libs/pybind11)
+    add_subdirectory(libs/pybind  ${CMAKE_BINARY_DIR}/bin/)
     add_subdirectory(python)
 endif(SLS_USE_PYTHON)
 
@@ -259,16 +259,13 @@ if(SLS_BUILD_DOCS)
     add_subdirectory(docs)
 endif(SLS_BUILD_DOCS)
 
-
 if(SLS_USE_MOENCH)
+    add_subdirectory(slsDetectorCalibration/tiffio)
     add_subdirectory(slsDetectorCalibration/moenchExecutables)
 endif(SLS_USE_MOENCH)
 
 if(SLS_MASTER_PROJECT)
-    # Set install dir CMake packages
     set(CMAKE_INSTALL_DIR "share/cmake/${PROJECT_NAME}")
-    # Set the list of exported targets
     set(PROJECT_LIBRARIES slsSupportShared slsDetectorShared slsReceiverShared)
-    # Generate and install package config file and version
     include(cmake/package_config.cmake)
 endif()

COPYING

@@ -0,0 +1,17 @@
+The SLS Detector Package is provided under:
+
+	SPDX-License-Identifier: LGPL-3.0-or-later
+
+Being under the terms of the GNU Lesser General Public License version 3 or later, 
+according with:
+
+	LICENSES/LGPL-3.0
+
+Source code under the Apache 2.0 License have the SPDX Identifier and are 
+according with:
+
+	LICENSES/ThirdParty/Apache-2.0
+
+All contributions to the SLS Detector Package are subject to this COPYING file.
+
+

LICENSES/GPL-3.0

@@ -1,3 +1,17 @@
+Valid-License-Identifier: GPL-3.0
+Valid-License-Identifier: GPL-3.0+
+SPDX-URL: https://spdx.org/licenses/GPL-3.0-or-later.html
+Usage-Guide:
+  To use this license in source code, put one of the following SPDX
+  tag/value pairs into a comment according to the placement
+  guidelines in the licensing rules documentation.
+  For 'GNU Library General Public License (LGPL) version 3.0 only' use:
+    SPDX-License-Identifier: GPL-3.0
+  For 'GNU Library General Public License (LGPL) version 3.0 or any later
+  version' use:
+    SPDX-License-Identifier: GPL-3.0-or-later
+License-Text:
+
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 

LICENSES/LGPL-3.0

@@ -1,3 +1,17 @@
+Valid-License-Identifier: LGPL-3.0
+Valid-License-Identifier: LGPL-3.0+
+SPDX-URL: https://spdx.org/licenses/LGPL-3.0-or-later.html
+Usage-Guide:
+  To use this license in source code, put one of the following SPDX
+  tag/value pairs into a comment according to the placement
+  guidelines in the licensing rules documentation.
+  For 'GNU Library General Public License (LGPL) version 3.0 only' use:
+    SPDX-License-Identifier: LGPL-3.0
+  For 'GNU Library General Public License (LGPL) version 3.0 or any later
+  version' use:
+    SPDX-License-Identifier: LGPL-3.0-or-later
+License-Text:
+
                    GNU LESSER GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 

LICENSES/Third Party/Apache-2.0

@@ -0,0 +1,210 @@
+Valid-License-Identifier: Apache-2.0
+SPDX-URL: https://spdx.org/licenses/Apache-2.0.html
+Usage-Guide:
+  To use this license in source code, put one of the following SPDX
+  tag/value pairs into a comment according to the placement
+  guidelines in the licensing rules documentation.
+    SPDX-License-Identifier: Apache-2.0
+License-Text:
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

cmake/FindQwt.cmake

@@ -1,118 +0,0 @@
-# Qt Widgets for Technical Applications
-# available at http://www.http://qwt.sourceforge.net/
-#
-# The module defines the following variables:
-#  QWT_FOUND - the system has Qwt
-#  QWT_INCLUDE_DIR - where to find qwt_plot.h
-#  QWT_INCLUDE_DIRS - qwt includes
-#  QWT_LIBRARY - where to find the Qwt library
-#  QWT_LIBRARIES - aditional libraries
-#  QWT_MAJOR_VERSION - major version
-#  QWT_MINOR_VERSION - minor version
-#  QWT_PATCH_VERSION - patch version
-#  QWT_VERSION_STRING - version (ex. 5.2.1)
-#  QWT_ROOT_DIR - root dir (ex. /usr/local)
-
-#=============================================================================
-# Copyright 2010-2013, Julien Schueller
-# All rights reserved.
-# 
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met: 
-# 
-# 1. Redistributions of source code must retain the above copyright notice, this
-#    list of conditions and the following disclaimer. 
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution. 
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# The views and conclusions contained in the software and documentation are those
-# of the authors and should not be interpreted as representing official policies, 
-# either expressed or implied, of the FreeBSD Project.
-#=============================================================================
-
-
-find_path ( QWT_INCLUDE_DIR
-  NAMES qwt_plot.h
-  HINTS $ENV{QWTDIR} $ENV{QWTDIR}/src ${QT_INCLUDE_DIR}
-  PATH_SUFFIXES qwt qwt-qt3 qwt-qt4 qwt-qt5
-)
-
-set ( QWT_INCLUDE_DIRS ${QWT_INCLUDE_DIR} )
-
-# version
-set ( _VERSION_FILE ${QWT_INCLUDE_DIR}/qwt_global.h )
-if ( EXISTS ${_VERSION_FILE} )
-  file ( STRINGS ${_VERSION_FILE} _VERSION_LINE REGEX "define[ ]+QWT_VERSION_STR" )
-  if ( _VERSION_LINE )
-    string ( REGEX REPLACE ".*define[ ]+QWT_VERSION_STR[ ]+\"(.*)\".*" "\\1" QWT_VERSION_STRING "${_VERSION_LINE}" )
-    string ( REGEX REPLACE "([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\1" QWT_MAJOR_VERSION "${QWT_VERSION_STRING}" )
-    string ( REGEX REPLACE "([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\2" QWT_MINOR_VERSION "${QWT_VERSION_STRING}" )
-    string ( REGEX REPLACE "([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\3" QWT_PATCH_VERSION "${QWT_VERSION_STRING}" )
-  endif ()
-endif ()
-
-
-# check version
-set ( _QWT_VERSION_MATCH TRUE )
-if ( Qwt_FIND_VERSION AND QWT_VERSION_STRING )
-  if ( Qwt_FIND_VERSION_EXACT )
-    if ( NOT Qwt_FIND_VERSION VERSION_EQUAL QWT_VERSION_STRING )
-      set ( _QWT_VERSION_MATCH FALSE )
-    endif ()
-  else ()
-    if ( QWT_VERSION_STRING VERSION_LESS Qwt_FIND_VERSION )
-      set ( _QWT_VERSION_MATCH FALSE )
-    endif ()
-  endif ()
-endif ()
-
-
-find_library ( QWT_LIBRARY
-  NAMES qwt qwt-qt3 qwt-qt4 qwt-qt5
-  HINTS $ENV{QWTDIR}/lib ${QT_LIBRARY_DIR}
-)
-
-set ( QWT_LIBRARIES ${QWT_LIBRARY} )
-
-
-# try to guess root dir from include dir
-if ( QWT_INCLUDE_DIR )
-  string ( REGEX REPLACE "(.*)/include.*" "\\1" QWT_ROOT_DIR ${QWT_INCLUDE_DIR} )
-# try to guess root dir from library dir
-elseif ( QWT_LIBRARY )
-  string ( REGEX REPLACE "(.*)/lib[/|32|64].*" "\\1" QWT_ROOT_DIR ${QWT_LIBRARY} )
-endif ()
-
-
-# handle the QUIETLY and REQUIRED arguments
-include ( FindPackageHandleStandardArgs )
-if ( CMAKE_VERSION LESS 2.8.3 )
-  find_package_handle_standard_args( Qwt DEFAULT_MSG QWT_LIBRARY QWT_INCLUDE_DIR _QWT_VERSION_MATCH )
-else ()
-  find_package_handle_standard_args( Qwt REQUIRED_VARS QWT_LIBRARY QWT_INCLUDE_DIR _QWT_VERSION_MATCH VERSION_VAR QWT_VERSION_STRING )
-endif ()
-
-
-mark_as_advanced (
-  QWT_LIBRARY 
-  QWT_LIBRARIES
-  QWT_INCLUDE_DIR
-  QWT_INCLUDE_DIRS
-  QWT_MAJOR_VERSION
-  QWT_MINOR_VERSION
-  QWT_PATCH_VERSION
-  QWT_VERSION_STRING
-  QWT_ROOT_DIR
-)

cmake/SlsAddFlag.cmake

@@ -0,0 +1,64 @@
+include(CheckCXXCompilerFlag)
+include(CheckCCompilerFlag)
+
+
+function(enable_cxx_warning flag target)
+    string(REPLACE "-W" "HAS_" flag_name ${flag})
+    check_cxx_compiler_flag(${flag} ${flag_name})
+    if(${flag_name})
+        target_compile_options(${target} INTERFACE ${flag})
+        message(STATUS "Adding: ${flag} to ${target}")
+    else()
+        message(STATUS "Flag: ${flag} not supported")
+    endif()
+endfunction()
+
+function(enable_c_warning flag target)
+    string(REPLACE "-W" "HAS_" flag_name ${flag})
+    check_c_compiler_flag(${flag} ${flag_name})
+    if(${flag_name})
+        target_compile_options(${target} INTERFACE ${flag})
+        message(STATUS "Adding: ${flag} to ${target}")
+    else()
+        message(STATUS "Flag: ${flag} not supported")
+    endif()
+endfunction()
+
+
+function(disable_cxx_warning flag target)
+    string(REPLACE "-W" "HAS_" flag_name ${flag})
+    check_cxx_compiler_flag(${flag} ${flag_name})
+
+    if(${flag_name})
+        string(REPLACE "-W" "-Wno-" neg_flag ${flag})
+        message(STATUS "Adding: ${neg_flag} to ${target}")
+        target_compile_options(${target} INTERFACE ${neg_flag})
+    else()
+        message(STATUS "Warning: ${flag} not supported no need to disable")
+    endif()
+endfunction()
+
+function(disable_c_warning flag target)
+    string(REPLACE "-W" "HAS_" flag_name ${flag})
+    check_c_compiler_flag(${flag} ${flag_name})
+    if(${flag_name})
+        string(REPLACE "-W" "-Wno-" neg_flag ${flag})
+        message(STATUS "Adding: ${neg_flag} to ${target}")
+        target_compile_options(${target} INTERFACE ${neg_flag})
+    else()
+        message(STATUS "Warning: ${flag} not supported no need to disable")
+    endif()
+endfunction()
+
+
+function(sls_disable_c_warning flag)
+    disable_c_warning(${flag} slsProjectCSettings)
+endfunction()
+
+function(sls_enable_cxx_warning flag)
+    enable_cxx_warning(${flag} slsProjectWarnings)
+endfunction()
+
+function(sls_disable_cxx_warning flag)
+    disable_cxx_warning(${flag} slsProjectWarnings)
+endfunction()

cmake/SlsFindZeroMQ.cmake

@@ -0,0 +1,38 @@
+function(custom_find_zmq)
+    set(ZeroMQ_HINT "" CACHE STRING "Hint where ZeroMQ could be found")
+    #Adapted from: https://github.com/zeromq/cppzmq/
+    if (NOT TARGET libzmq)
+        if(ZeroMQ_HINT)
+            message(STATUS "Looking for ZeroMQ in: ${ZeroMQ_HINT}")
+            find_package(ZeroMQ 4 
+                NO_DEFAULT_PATH
+                HINTS ${ZeroMQ_HINT}
+            )
+        else()
+            find_package(ZeroMQ 4 QUIET)
+        endif()
+        
+    # libzmq autotools install: fallback to pkg-config
+    if(ZeroMQ_FOUND)
+        message(STATUS "Found libzmq using find_package")
+    else()
+        message(STATUS "CMake libzmq package not found, trying again with pkg-config (normal install of zeromq)")
+        list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake/libzmq-pkg-config)
+        find_package(ZeroMQ 4 REQUIRED)
+    endif()
+
+    # TODO "REQUIRED" above should already cause a fatal failure if not found, but this doesn't seem to work
+    if(NOT ZeroMQ_FOUND)
+        message(FATAL_ERROR "ZeroMQ was not found, neither as a CMake package nor via pkg-config")
+    endif()
+
+    if (ZeroMQ_FOUND AND NOT TARGET libzmq)
+        message(FATAL_ERROR "ZeroMQ version not supported!")
+    endif()
+    endif()
+
+    get_target_property(VAR libzmq IMPORTED_LOCATION)
+    message(STATUS "Using libzmq: ${VAR}")
+
+
+endfunction()

cmake/libzmq-pkg-config/FindZeroMQ.cmake

@@ -0,0 +1,36 @@
+#From: https://github.com/zeromq/cppzmq/
+set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON)
+find_package(PkgConfig)
+pkg_check_modules(PC_LIBZMQ QUIET libzmq)
+
+set(ZeroMQ_VERSION ${PC_LIBZMQ_VERSION})
+
+find_path(ZeroMQ_INCLUDE_DIR zmq.h
+        PATHS ${ZeroMQ_DIR}/include
+        ${PC_LIBZMQ_INCLUDE_DIRS}
+)
+
+find_library(ZeroMQ_LIBRARY
+        NAMES zmq
+        PATHS ${ZeroMQ_DIR}/lib
+        ${PC_LIBZMQ_LIBDIR}
+        ${PC_LIBZMQ_LIBRARY_DIRS}
+)
+
+if(ZeroMQ_LIBRARY OR ZeroMQ_STATIC_LIBRARY)
+    set(ZeroMQ_FOUND ON)
+    message(STATUS "Found libzmq using PkgConfig")
+endif()
+
+set ( ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} )
+set ( ZeroMQ_INCLUDE_DIRS ${ZeroMQ_INCLUDE_DIR} )
+
+if (NOT TARGET libzmq)
+    add_library(libzmq UNKNOWN IMPORTED)
+    set_target_properties(libzmq PROPERTIES
+            IMPORTED_LOCATION ${ZeroMQ_LIBRARIES}
+            INTERFACE_INCLUDE_DIRECTORIES ${ZeroMQ_INCLUDE_DIRS})
+endif()
+
+include ( FindPackageHandleStandardArgs )
+find_package_handle_standard_args ( ZeroMQ DEFAULT_MSG ZeroMQ_LIBRARIES ZeroMQ_INCLUDE_DIRS )

cmake/package_config.cmake

@@ -26,7 +26,7 @@ install(FILES
 )
 
 install(FILES
-  "${CMAKE_SOURCE_DIR}/libzmq-pkg-config/FindZeroMQ.cmake"
+  "${CMAKE_SOURCE_DIR}/cmake/libzmq-pkg-config/FindZeroMQ.cmake"
   COMPONENT devel
   DESTINATION ${CMAKE_INSTALL_DIR}/libzmq-pkg-config
 )

cmk.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 CMAKE="cmake3"
 BUILDDIR="build"
 INSTALLDIR=""
@@ -16,6 +18,7 @@ CTBGUI=0
 MANUALS=0
 MANUALS_ONLY_RST=0
 MOENCHZMQ=0
+ZMQ_HINT_DIR=""
 
 
 CLEAN=0
@@ -24,25 +27,26 @@ CMAKE_PRE=""
 CMAKE_POST=""
 
 usage() { echo -e "
-Usage: $0 [-c] [-b] [-p] [e] [t] [r] [g] [s] [u] [i] [m] [n] [-h] [z] [-d <HDF5 directory>] [-l Install directory] [-k <CMake command>] [-j <Number of threads>]
+Usage: $0 [-b] [-c] [-d <HDF5 directory>] [e] [g] [-h] [i] [-j <Number of threads>] [-k <CMake command>] [-l <Install directory>] [m] [n] [-p] [-q <Zmq hint directory>] [r] [s] [t] [u] [z]  
  -[no option]: only make
- -c: Clean
  -b: Builds/Rebuilds CMake files normal mode
- -p: Builds/Rebuilds Python API
- -h: Builds/Rebuilds Cmake files with HDF5 package
+ -c: Clean
  -d: HDF5 Custom Directory
+ -e: Debug mode
+ -g: Build/Rebuilds only gui
+ -h: Builds/Rebuilds Cmake files with HDF5 package
+ -i: Builds tests
+ -j: Number of threads to compile through
  -k: CMake command
  -l: Install directory
- -t: Build/Rebuilds only text client
- -r: Build/Rebuilds only receiver
- -g: Build/Rebuilds only gui
- -s: Simulator
- -u: Chip Test Gui
- -j: Number of threads to compile through
- -e: Debug mode
- -i: Builds tests
  -m: Manuals
  -n: Manuals without compiling doxygen (only rst)
+ -p: Builds/Rebuilds Python API
+ -q: Zmq hint directory
+ -r: Build/Rebuilds only receiver
+ -s: Simulator
+ -t: Build/Rebuilds only text client
+ -u: Chip Test Gui
  -z: Moench zmq processor
 
 Rebuild when you switch to a new build and compile in parallel:
@@ -79,69 +83,50 @@ For rebuilding only certain sections
  
  " ; exit 1; }
 
-while getopts ":bpchd:k:l:j:trgeisumnz" opt ; do
+while getopts ":bcd:eghij:k:l:mnpq:rstuz" opt ; do
 	case $opt in
 	b) 
 		echo "Building of CMake files Required"
 		REBUILD=1
 		;;
-	p)
-    	echo "Compiling Options: Python" 
-		PYTHON=1
-		REBUILD=1
-		;;   
 	c) 
 		echo "Clean Required"
 		CLEAN=1
 		;;
-	h) 
-		echo "Building of CMake files with HDF5 option Required"
-		HDF5=1
-		REBUILD=1
-		;;
 	d) 
 		echo "New HDF5 directory: $OPTARG" 
 		HDF5DIR=$OPTARG
 		;;
-	l)
-		echo "CMake install directory: $OPTARG"
-		INSTALLDIR="$OPTARG"
+	e)
+		echo "Compiling Options: Debug" 
+		DEBUG=1
+		;;  
+	g) 
+		echo "Compiling Options: GUI" 
+		GUI=1
+		REBUILD=1
+		;; 
+	h) 
+		echo "Building of CMake files with HDF5 option Required"
+		HDF5=1
+		REBUILD=1
+		;;
+	i)
+		echo "Compiling Options: Tests" 
+		TESTS=1
+		;;   
+	j) 
+		echo "Number of compiler threads: $OPTARG" 
+		COMPILERTHREADS=$OPTARG
 		;;
 	k)
 		echo "CMake command: $OPTARG"
 		CMAKE="$OPTARG"
 		;;
-	j) 
-		echo "Number of compiler threads: $OPTARG" 
-		COMPILERTHREADS=$OPTARG
+	l)
+		echo "CMake install directory: $OPTARG"
+		INSTALLDIR="$OPTARG"
 		;;
-	t) 
-    	echo "Compiling Options: Text Client" 
-		TEXTCLIENT=1
-		REBUILD=1
-		;;      
-	r) 
-		echo "Compiling Options: Receiver" 
-		RECEIVER=1
-		REBUILD=1
-		;;      
-	g) 
-		echo "Compiling Options: GUI" 
-		GUI=1
-		REBUILD=1
-		;;  
-	e)
-		echo "Compiling Options: Debug" 
-		DEBUG=1
-		;;   
-	i)
-		echo "Compiling Options: Tests" 
-		TESTS=1
-		;;   
-	s)
-		echo "Compiling Options: Simulator" 
-		SIMULATOR=1
-		;; 
 	m)	
 		echo "Compiling Manuals"
 		MANUALS=1
@@ -150,14 +135,37 @@ while getopts ":bpchd:k:l:j:trgeisumnz" opt ; do
 		echo "Compiling Manuals (Only RST)"
 		MANUALS_ONLY_RST=1
 		;;
-	z)	
-		echo "Compiling Moench Zmq Processor"
-		MOENCHZMQ=1
+	p)
+    	echo "Compiling Options: Python" 
+		PYTHON=1
+		REBUILD=1
+		;;   
+	q) 
+		echo "Zmq hint directory: $OPTARG" 
+		ZMQ_HINT_DIR=$OPTARG
+		;;
+	r) 
+		echo "Compiling Options: Receiver" 
+		RECEIVER=1
+		REBUILD=1
+		;;      
+	s)
+		echo "Compiling Options: Simulator" 
+		SIMULATOR=1
+		;; 
+	t) 
+    	echo "Compiling Options: Text Client" 
+		TEXTCLIENT=1
+		REBUILD=1
 		;;
 	u)
 		echo "Compiling Options: Chip Test Gui"
 		CTBGUI=1
 		;;
+	z)	
+		echo "Compiling Moench Zmq Processor"
+		MOENCHZMQ=1
+		;;
   \?)
     echo "Invalid option: -$OPTARG" 
 		usage
@@ -252,6 +260,12 @@ if [ $TESTS -eq 1 ]; then
 	echo "Tests Option enabled"
 fi 
 
+#zmq hint dir
+if [ -n "$ZMQ_HINT_DIR" ]; then
+	CMAKE_POST+=" -DZeroMQ_HINT="$ZMQ_HINT_DIR
+	CMAKE_POST+=" -DZeroMQ_DIR="
+#	echo "Enabling Zmq Hint Directory: $ZMQ_HINT_DIR"
+fi 
 
 #hdf5 rebuild
 if [ $HDF5 -eq 1 ]; then

conda-recepie/build.sh

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 
 mkdir build
 mkdir install
@@ -19,4 +21,4 @@ echo "Building using: ${NCORES} cores"
 cmake --build . -- -j${NCORES}
 cmake --build . --target install
 
-CTEST_OUTPUT_ON_FAILURE=1 ctest -j 2
+CTEST_OUTPUT_ON_FAILURE=1 ctest -j 1

conda-recepie/build_pylib.sh

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 
 echo "|<-------- starting python build"
 cd python

conda-recepie/conda_build_config.yaml

@@ -1,8 +1,8 @@
 python:
-    - 3.6
-    - 3.7
     - 3.8
     - 3.9
+    - 3.10
+    - 3.11
     
 numpy: 
-    - 1.17
+    - 1.17

conda-recepie/copy_ctbgui.sh

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 mkdir $PREFIX/lib
 mkdir $PREFIX/bin
 mkdir $PREFIX/include

conda-recepie/copy_gui.sh

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 #Copy the GUI 
 mkdir -p $PREFIX/bin
 cp build/install/bin/slsDetectorGui $PREFIX/bin/.

conda-recepie/copy_lib.sh

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 
 mkdir -p $PREFIX/lib
 mkdir -p $PREFIX/bin
@@ -17,4 +19,4 @@ cp build/install/bin/slsMultiReceiver $PREFIX/bin/.
 
 
 cp build/install/include/sls/* $PREFIX/include/sls
-cp -r build/install/share/ $PREFIX/share
+cp -rv build/install/share $PREFIX

conda-recepie/copy_moench.sh

@@ -1,4 +1,6 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
+
 #Copy the Moench executables 
 mkdir -p $PREFIX/bin
-cp build/install/bin/moench04ZmqProcess $PREFIX/bin/.
-cp build/install/bin/moenchZmqProcess $PREFIX/bin/.
+cp build/install/bin/moench* $PREFIX/bin/.

conda-recepie/meta.yaml

@@ -17,8 +17,7 @@ requirements:
     - {{ compiler('c') }}
     - {{compiler('cxx')}}
     - cmake
-    - qwt 6.*
-    - qt 4.8.*
+    - qt 5.*
     - zeromq
     - xorg-libx11
     - xorg-libice
@@ -111,12 +110,10 @@ outputs:
         - {{ compiler('c') }}
         - {{compiler('cxx')}}
         - {{ pin_subpackage('slsdetlib', exact=True) }}
-        - qwt 6.*
 
       run:
         - {{ pin_subpackage('slsdetlib', exact=True) }}
-        - qwt 6.*
-        - qt 4.8.*
+        - qt 5.*
         - expat
 
   - name: moenchzmq

conda-recepie/run_test.sh

@@ -1 +1,3 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 ctest -j2

ctbGui/CMakeLists.txt

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 
 
 find_package(ROOT CONFIG REQUIRED  COMPONENTS Core Gui)
@@ -32,7 +34,7 @@ add_executable(ctbGui
     ctbAdcs.cpp
     ctbPattern.cpp
     ctbAcquisition.cpp
-    ${CMAKE_SOURCE_DIR}/slsDetectorCalibration/tiffIO.cpp
+    ${CMAKE_SOURCE_DIR}/slsDetectorCalibration/tiffio/src/tiffIO.cpp
 )
 
 
@@ -41,6 +43,7 @@ target_include_directories(ctbGui PRIVATE
     ${CMAKE_SOURCE_DIR}/slsDetectorCalibration/dataStructures
     ${CMAKE_SOURCE_DIR}/slsDetectorCalibration/interpolations
     ${CMAKE_SOURCE_DIR}/slsDetectorCalibration/
+    ${CMAKE_SOURCE_DIR}/slsDetectorCalibration/tiffio/include/
 )
 
 # Headders needed for ROOT dictionary generation
@@ -59,7 +62,6 @@ set( HEADERS
 #set(ROOT_INCLUDE_PATH ${CMAKE_CURRENT_SOURCE_DIR})
 
 # ROOT dictionary generation 
-include("${ROOT_DIR}/RootMacros.cmake")
 root_generate_dictionary(ctbDict ${HEADERS} LINKDEF ctbLinkDef.h)
 add_library(ctbRootLib SHARED ctbDict.cxx)
 target_include_directories(ctbRootLib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
@@ -84,4 +86,5 @@ target_link_libraries(ctbGui PUBLIC
 set_target_properties(ctbGui PROPERTIES
     RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin
 
-)
+)
+

ctbGui/Makefile.root5

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 
 
 INCS=ctbMain.h ctbDacs.h   ctbPattern.h  ctbSignals.h  ctbAdcs.h ctbAcquisition.h ctbPowers.h   ctbSlowAdcs.h 

ctbGui/Makefile.root6

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 
 
 INCS=ctbMain.h ctbDacs.h   ctbPattern.h  ctbSignals.h  ctbAdcs.h ctbAcquisition.h ctbPowers.h   ctbSlowAdcs.h  

ctbGui/ctbAcquisition.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 //#define TESTADC
 
 
@@ -826,14 +828,14 @@ void ctbAcquisition::setCanvas(TCanvas* c) {
   myCanvas->AddExec("dynamic",Form("((ctbAcquisition*)%p)->canvasClicked()",this));
   // myCanvas->AddExec("ex","canvasClicked()");
 }
-void ctbAcquisition::dataCallback(detectorData *data, long unsigned int index, unsigned int dum, void* pArgs) {
+void ctbAcquisition::dataCallback(sls::detectorData *data, long unsigned int index, unsigned int dum, void* pArgs) {
 
   // return 
   ((ctbAcquisition*)pArgs)->plotData(data,index);
 }
 
 
-int ctbAcquisition::plotData(detectorData *data, int index) {
+int ctbAcquisition::plotData(sls::detectorData *data, int index) {
 
   /*
 ******************************************************************
@@ -986,7 +988,7 @@ sample1 (dbit0 + dbit1 +...)if (cmd == "rx_dbitlist") {
 	ped=0;
 	aval=dataStructure->getValue(data->data,x,y);
 	//aval=dataStructure->getChannel(data->data,x,y);
-	cout << x << " " <<y << " "<< aval << endl;
+	// cout << x << " " <<y << " "<< aval << endl;
 	if (cbGetPedestal->IsOn()) {
 	  if (photonFinder) {
 	    photonFinder->addToPedestal(aval,x,y);

ctbGui/ctbAcquisition.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #ifndef CTBACQUISITION_H
 #define CTBACQUISITION_H
 #include  <TGFrame.h>
@@ -26,8 +28,8 @@ class TGTextButton;
 namespace sls
 {
    class Detector;
+   class detectorData;
 };
-class detectorData;
 
 template <class dataType> class slsDetectorData;
 
@@ -199,10 +201,10 @@ class ctbAcquisition : public TGGroupFrame {
    void setBitGraph (int i ,int en, Pixel_t col);
    void startAcquisition();
    static   void progressCallback(double,void*);
-   static void dataCallback(detectorData*, long unsigned int, unsigned int,  void*);
+   static void dataCallback(sls::detectorData*, long unsigned int, unsigned int,  void*);
    int StopFlag;
    
-   int plotData(detectorData*, int);
+   int plotData(sls::detectorData*, int);
 
    void setPatternFile(const char* t);
 

ctbGui/ctbAdcs.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #include <TApplication.h>
 #include <TGClient.h>
 #include <TCanvas.h>

ctbGui/ctbAdcs.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 
 
 

ctbGui/ctbDacs.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 
 #include <stdio.h>
 #include <iostream>

ctbGui/ctbDacs.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 
 
 #ifndef CTBDACS_H

ctbGui/ctbDefs.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #pragma once
 
 #include <string>

ctbGui/ctbGui.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #include <TApplication.h>
 #include <TColor.h>
 

ctbGui/ctbLinkDef.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #pragma link C++ class ctbMain;
 #pragma link C++ class ctbDacs;
 #pragma link C++ class ctbDac;

ctbGui/ctbMain.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #include <TApplication.h>
 #include <TGClient.h>
 #include <TCanvas.h>

ctbGui/ctbMain.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #ifndef CTBMAIN_H
 #define CTBMAIN_H
 #include <TGFrame.h>

ctbGui/ctbPattern.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #include <TApplication.h>
 #include <TGClient.h>
 #include <TCanvas.h>

ctbGui/ctbPattern.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #ifndef CTBPATTERN_H
 #define CTBPATTERN_H
 #include  <TGFrame.h>

ctbGui/ctbPowers.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #include  <TGFrame.h>
 
 

ctbGui/ctbPowers.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #ifndef CTBPOWERS_H
 #define CTBPOWERS_H
 

ctbGui/ctbSignals.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #include <TApplication.h>
 #include <TGClient.h>
 #include <TCanvas.h>

ctbGui/ctbSignals.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #ifndef CTBSIGNALS_H
 #define CTBSIGNALS_H
 #include  <TGFrame.h>

ctbGui/ctbSlowAdcs.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 
 #include <stdio.h>
 #include <iostream>

ctbGui/ctbSlowAdcs.h

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 
 
 #ifndef CTBSLOWADCS_H

ctbGui/patternGenerator/deserializer.cpp

@@ -1,110 +0,0 @@
-
-#include <stdlib.h>  
-#include <stdint.h>  
-#include <string.h>  
-#include <sys/utsname.h>   
-#include <sys/types.h>
-#include <unistd.h>  
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <errno.h>
-#include <math.h>
-#include <fcntl.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-
-int main(int argc, char *argv[]) {
-
-  int iarg;
-  char fname[10000];
-  uint64_t word;
-  int val[64];
-  int bit[64];
-  FILE *fdin;
-  
-  int nb=2;
-  int off=0;
-  int ioff=0;
-  int dr=24;
-  int idr=0;
-  int ib=0;
-  int iw=0;
-  bit[0]=19;
-  bit[1]=8;
-  //  for (iarg=0; iarg<argc; iarg++) printf("%d %s\n",iarg, argv[iarg]);
-  
-  if (argc<2) printf("Error: usage is %s fname [dr off  b0 b1 bn]\n");
-  
-  if (argc>2) dr=atoi(argv[2]);
-  if (argc>3) off=atoi(argv[3]);
-  if (argc>4) {
-  for (ib=0; ib<64; ib++) {
-    if (argc>4+ib) {
-      bit[ib]=atoi(argv[4+ib]);
-      nb++;
-    }
-  }
-  
-  }
-  
-  idr=0;
-  for (ib=0; ib<nb; ib++) {
-    val[ib]=0;
-  }
-  
-  
-  fdin=fopen(argv[1],"rb");
-  if (fdin==NULL) {
-    printf("Cannot open input file %s for reading\n",argv[1]);
-    return 200;
-  } 
-  
-  while (fread((void*)&word, 8, 1, fdin)) {
-    //   printf("%llx\n",word);
-    if (ioff<off) ioff++;
-    else {
-      
-      for (ib=0; ib<nb; ib++) {
-	if (word&(1<<bit[ib])) 	  val[ib]|=(1<<idr);
-      }
-      idr++;
-      if (idr==dr) {
-	idr=0;
-	fprintf(stdout,"%d\t",iw++);
-	for (ib=0; ib<nb; ib++) {
-#ifdef HEX
-	  fprintf(stdout,"%08llx\t",val[ib]);
-#else
-	  fprintf(stdout,"%lld\t",val[ib]);
-	  
-#endif
-	  
-	  val[ib]=0;
-	}
-	fprintf(stdout,"\n");
-      }
-      
-    }
-
-  }
-  if (idr!=0) {
-    fprintf(stdout,"%d\t",iw++);
-    for (ib=0; ib<nb; ib++) {
-#ifdef HEX
-	  fprintf(stdout,"%08llx\t",val[ib]);
-#else
-	  fprintf(stdout,"%lld\t",val[ib]);
-	  
-#endif
-	  
-	  val[ib]=0;
-    }
-    fprintf(stdout,"\n");
-  }
-
-  fclose(fdin);
-
-  return 0;
-}

ctbGui/patternGenerator/generator.c

@@ -1,177 +0,0 @@
-/****************************************************************************
-usage to generate a patter test.pat from test.p
-
-gcc -DINFILE="\"test.p\"" -DOUTFILE="\"test.pat\"" -o test.exe generator.c ; ./test.exe ; rm test.exe
-
-
-*************************************************************************/
-
-#include <stdlib.h>  
-#include <stdint.h>  
-#include <string.h>  
-#include <sys/utsname.h>   
-#include <sys/types.h>
-#include <unistd.h>  
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <errno.h>
-#include <math.h>
-#include <fcntl.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-
-#define MAXLOOPS 3
-#define MAXTIMERS 3
-#define MAXWORDS 1024
-
-
-
-uint64_t pat=0;
-uint64_t iopat=0;
-uint64_t clkpat=0;
-
-int iaddr=0;
-int waitaddr[3]={MAXWORDS,MAXWORDS,MAXWORDS};
-int startloopaddr[3]={MAXWORDS,MAXWORDS,MAXWORDS}; 
-int stoploopaddr[3]={MAXWORDS,MAXWORDS,MAXWORDS}; 
-int start=0, stop=0;
-uint64_t waittime[3]={0,0,0};
-int nloop[3]={0,0,0};
-
-char infile[10000], outfile[10000];
-
-FILE *fd, *fd1;
-uint64_t PAT[MAXWORDS];
-
-
-int i,ii,iii,j,jj,jjj,pixx,pixy,memx,memy,muxout,memclk,colclk,rowclk,muxclk,memcol,memrow,loopcounter;
-
-void setstart() {
-  start=iaddr;
-}
-
-void setstop() {
-  stop=iaddr;
-}
-
-void setinput(int bit) {
-  uint64_t mask=1;
-  mask=mask<<bit;
-  iopat &= ~mask;
-}
-
-void setoutput(int bit) {
-  uint64_t mask=1;
-  mask=mask<<bit;
-  iopat |= mask;
-}
-
-void setclk(int bit) {
-  uint64_t mask=1;
-  mask=mask<<bit;
-  iopat |= mask;
-  clkpat |= mask;
-}
-
-void clearbit(int bit){
-  uint64_t mask=1;
-  mask=mask<<bit;
-  pat &= ~mask;
-}
-void setbit(int bit){
-  uint64_t mask=1;
-  mask=mask<<bit;
-  pat |= mask;
-}
-
-int checkbit(int bit) {
-  uint64_t mask=1;
-  mask=mask<<bit;
-  return (pat & mask ) >>bit;
-}
-
-void setstartloop(int iloop) {
-  if (iloop>=0 && iloop<MAXLOOPS)
-    startloopaddr[iloop]=iaddr;
-}
-
-
-void setstoploop(int iloop) {
-  if (iloop>=0 && iloop<MAXLOOPS)
-    stoploopaddr[iloop]=iaddr;
-}
-
-
-void setnloop(int iloop, int n) {
-  if (iloop>=0 && iloop<MAXLOOPS)
-    nloop[iloop]=n;
-}
-
-void setwaitpoint(int iloop) {
-  if (iloop>=0 && iloop<MAXTIMERS)
-    waitaddr[iloop]=iaddr;
-}
-
-
-void setwaittime(int iloop, uint64_t t) {
-  if (iloop>=0 && iloop<MAXTIMERS)
-    waittime[iloop]=t;
-}
-
-
-
-
-void pw(){
-  if (iaddr<MAXWORDS)
-    PAT[iaddr]= pat;
-  fprintf(fd,"patword 0x%04x 0x%016llx\n",iaddr, pat);
-  iaddr++;
-  if (iaddr>=MAXWORDS) printf("ERROR: too many word in the pattern (%d instead of %d)!",iaddr, MAXWORDS);
-}
-
-int parseCommand(int clk, int cmdbit, int cmd, int length) {
-  int ibit;
-  clearbit(clk);
-  for (ibit=0; ibit<length; ibit++) {
-    if (cmd&(1>>ibit))
-      setbit(cmdbit);
-    else
-      clearbit(cmdbit);
-    pw();
-    /******/
-    setbit(clk);
-    pw();
-    /******/
-  }
-};
-
-
-
-main(void) {
-  int iloop=0;
-  fd=fopen(OUTFILE,"w");
-#include INFILE
-
-  fprintf(fd,"patioctrl 0x%016llx\n",iopat);
-  fprintf(fd,"patclkctrl 0x%016llx\n",clkpat);
-  fprintf(fd,"patlimits 0x%04x 0x%04x\n",start, stop);
-
-  for (iloop=0; iloop<MAXLOOPS; iloop++) {
-    fprintf(fd,"patloop%d 0x%04x 0x%04x\n",iloop, startloopaddr[iloop], stoploopaddr[iloop]);
-    if ( startloopaddr[iloop]<0 || stoploopaddr[iloop]<= startloopaddr[iloop]) nloop[iloop]=0;
-    fprintf(fd,"patnloop%d %d\n",iloop, nloop[iloop]);
-  }
-
-  for (iloop=0; iloop<MAXTIMERS; iloop++) {
-    fprintf(fd,"patwait%d 0x%04x\n",iloop, waitaddr[iloop]);
-    if (waitaddr[iloop]<0) waittime[iloop]=0;
-    fprintf(fd,"patwaittime%d %lld\n",iloop, waittime[iloop]);
-  }
-  
-  close((int)fd);
-  fd1=fopen(OUTFILEBIN,"w");
-  fwrite(PAT,sizeof(uint64_t),iaddr, fd1);
-  close((int)fd1);
-}

ctbGui/patternGenerator/test.p

@@ -1,201 +0,0 @@
-//define signals and directions (Input, outputs, clocks)
-
-
-#define compTestIN 1
-setoutput(compTestIN);
-
-#define curON 32
-setoutput(curON);
-
-#define side_clk 2
-setclk(side_clk);
-
-#define side_din 3
-setoutput(side_din);
-
-#define clear_shr 4
-setoutput(clear_shr);
-
-#define bottom_din 5
-setoutput(bottom_din);
-
-#define bottom_clk 6
-setclk(bottom_clk);
-
-#define gHG 7
-setoutput(gHG);
-
-#define bypassCDS 31
-setoutput(bypassCDS);
-
-
-#define ENprechPRE 8
-setoutput(ENprechPRE);
-
-
-#define res 9
-setoutput(res);
-
-#define pulseOFF 30
-setoutput(pulseOFF);
-
-#define connCDS 27
-setoutput(connCDS);
-
-#define Dsg_1 24
-setoutput(Dsg_1);
-
-
-#define Dsg_2 25
-setoutput(Dsg_2);
-
-
-#define Dsg_3 23
-setoutput(Dsg_3);
-
-#define sto0 10
-setoutput(sto0);
-
-#define sto1 11
-setoutput(sto1);
-
-#define sto2 12
-setoutput(sto2);
-
-#define resCDS 13
-setoutput(resCDS);
-
-#define prechargeConnect 14
-setoutput(prechargeConnect);
-
-#define pulse 15
-setoutput(pulse);
-
-#define PCT_mode 21
-setoutput(PCT_mode);
-
-#define res_DGS 16
-setoutput(res_DGS);
-
-#define adc_ena 17
-setoutput(adc_ena);
-
-
-#define CLKBIT 18
-setclk(CLKBIT);
-
-
-#define adc_sync 63
-setoutput(adc_sync);
-
-
-
-
-
-	  
-#define PW pw()
-#define SB(x) setbit(x) 
-#define CB(x) clearbit(x)
-#define CLOCK  clearbit(CLKBIT); pw();setbit(CLKBIT);pw()
-#define LCLOCK  clearbit(CLKBIT); pw();setbit(CLKBIT);pw();clearbit(CLKBIT); pw()
-#define CLOCKS(x) for (i=0;i<x;i++) {clearbit(CLKBIT);pw(); setbit(CLKBIT); pw();}
-#define STOP setstop();
-#define START setstart();
-#define REPEAT(x) for (i=0;i<(x);i++) {pw();}
-#define DOFOR(x) for (j=0;j<(x);j++) {
-// }				     
-#define STARTUP1 CB(compTestIN);SB(clear_shr);CB(side_clk);CB(side_din);CB(bottom_din);CB(bottom_clk);
-#define STARTUP2 CB(pulse);SB(PCT_mode);SB(pulseOFF);CB(curON);
-#define STARTUP3 SB(res);SB(gHG);SB(ENprechPRE);
-#define STARTUP4 SB(bypassCDS); CB(connCDS);CB(sto0);SB(sto1);SB(sto2);
-#define STARTUP5 SB(resCDS);CB(Dsg_1);CB(Dsg_2);SB(Dsg_3);CB(prechargeConnect);SB(res_DGS);
-#define STARTUP STARTUP1 STARTUP2 STARTUP3 STARTUP4 STARTUP5 PW;
-
-
-
-
-
-//****NOTES****//
-//FUNCTIONS
-//Declare functions at the beginning 
-void load_pix(int nx, int ny)
-{//SELECT PIXEL 1,1 for readout
-SB(clear_shr);PW;PW;
-CB(clear_shr);PW;PW;PW;PW;
-
-SB(side_din);PW;
-SB(side_clk);PW;
-CB(side_din);
-setstartloop(0); //loop on the rows
-SB(side_clk);PW;
-setstoploop(0);  //finish loop on the rows
-setnloop(0,ny); //set number row selected -can be changed dynamically
-CB(side_clk);PW;
-SB(bottom_din);PW;
-SB(bottom_clk);PW;
-CB(bottom_din);
-setstartloop(1); //loop on the columns
-SB(bottom_clk);PW;
-setstoploop(1); //loop on the columns
-setnloop(1,ny); //set number columns selected -can be changed dynamically
-}
-   
-void load_col(void)
-{//SELECT COLUMN 1 for readout
-SB(clear_shr);PW;PW;
-CB(clear_shr);PW;PW;PW;PW;
-SB(bottom_din);PW;
-SB(bottom_clk);PW;
-CB(bottom_clk);PW;
-CB(bottom_din);PW;
-}
-//END of FUNCTIONS
-////////////////////////////////////////////////////////
-//LET BYPASS PREAMP AND CDS and write on preamp out.//
-//THIS ALLOWS CHECKING SOURCE FOLLOWERS               //
-////////////////////////////////////////////////////////
-
-   
-PW;
-   
-SB(5); PW;
-
-CB(5); PW;
-
-START; //pattern starts from here
-STARTUP;
-setwaitpoint(0); //set wait points
-PW;
-setwaittime(0,20); //wait time - can be changed dynamically
-SB(adc_ena);PW;
-   printf("ADC sync %x %d %llx\n",iaddr,adc_sync, pat);
-SB(adc_sync);PW;
-   printf("ADC sync %x %d  %llx\n",iaddr, adc_sync, pat);
-CB(gHG);
-setwaitpoint(1); //set wait points
-setwaittime(1,16); //wait time - can be changed dynamically
-CB(adc_sync);PW;
-load_pix(10, 20);
-
-CB(res);
-//CB(Dsg_3);PW;
-CB(res_DGS);
-setwaitpoint(2); //set wait points
-setwaittime(2,1000); //wait time - can be changed dynamically
-
-//SB(res_DGS);
-//PW;
-//SB(Dsg_3);
-//
-//CB(connCDS);
-//TEST SIGNALS END
-//
-REPEAT(20)
-
-//****************//
-//*FINAL COMMANDS*//
-//****************//
-CB(adc_ena);PW;
-//STARTUP;
-STOP; PW; //stops here
-//REPEAT(4);

docs/CMakeLists.txt

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 find_package(Doxygen REQUIRED)
 find_package(Sphinx REQUIRED)
 
@@ -53,6 +55,9 @@ set(SPHINX_SOURCE_FILES
   src/troubleshooting.rst
   src/receivers.rst
   src/slsreceiver.rst
+  src/udpheader.rst
+  src/udpconfig.rst
+  src/udpdetspec.rst
 )
 
 foreach(filename ${SPHINX_SOURCE_FILES})

docs/src/dependencies.rst

@@ -21,10 +21,8 @@ the shared libraries these are needed:
 GUI
 -----------------------
 
-The GUI is currently using Qt4 but watch out for an upgrade to 5. 
-
- * Qt 4.8
- * Qwt 6.1
+ * Qt 5.9
+ * Qwt 6.1.5 (packaged in libs/)
 
 -----------------------
 Python bindings
@@ -34,6 +32,12 @@ Python bindings
  * pybind11 (packaged in libs/)
 
 
+-----------------------
+Moench executables
+-----------------------
+
+ * libtiff
+
 -----------------------
 Documentation
 -----------------------

docs/src/firmware.rst

@@ -1,8 +1,6 @@
 Firmware Upgrade
 =================
 
-
-
 Eiger
 -------------
 
@@ -18,30 +16,9 @@ Upgrade
 ^^^^^^^^
 #. Tftp must be already installed on your pc to use the bcp script.
 
-#. Kill the on-board servers and copy new servers to the board. 
+#. Copy new servers to the board. See :ref:`how to upgrade detector servers<Detector Server Upgrade>` for more detals. A reboot should have started the new linked servers automatically. For Eiger, do not reboot yet as we need to program the firmware via bit files.
 
-    .. code-block:: bash
-
-        # Option 1: from detector console
-        # kill old server
-        ssh root@bebxxx
-        killall eigerDetectorServer
-
-        # copy new server
-        cd executables
-        scp user@pc:/path/eigerDetectorServerxxx .
-        chmod 777 eigerDetectorServerxxx
-        ln -sf eigerDetectorServerxxx eigerDetectorServer
-        sync
-
-        # Options 2: from client console for multiple modules
-        for i in bebxxx bebyyy;
-        do ssh root@$i killall eigerDetectorServer;
-        scp eigerDetectorServerxxx root@$i:~/executables/eigerDetectorServer;
-        ssh root@$i sync; done
-
-
-    * This is crucial when registers between firmwares change. Failure to do so will result in linux on boards to crash and boards can't be pinged anymore.
+    * This step is crucial when registers between firmwares change. Failure to do so will result in linux on boards to crash and boards can't be pinged anymore.
 
 #. Bring the board into programmable mode using either of the 2 ways. Both methods result in only the central LED blinking.
     
@@ -50,8 +27,13 @@ Upgrade
         Do a hard reset for each half module on back panel boards, between the LEDs, closer to each of the 1G ethernet connectors. Push until all LEDs start to blink.
     
     * Software:  
+
         .. code-block:: bash
 
+            # Option 1: if the old server is still running:
+            sls_detector_put execcommand "./boot_recovery"
+
+            # Option 2:
             ssh root@bebxxx
             cd executables
             ./boot_recovery
@@ -79,11 +61,24 @@ Upgrade
         #update front right fpga
         bcp download.bit bebxxx:/febr
 
-        #update kernel (only if required by the SLS Detector Group)
+        #update kernel (only if required by us)
         bcp download.bit bebxxx:/kernel
 
 #. Reboot the detector.
 
+    .. code-block:: bash
+
+        # In the first terminal where we saw "Succeess"
+        # reconfig febX is necessary only if you have flashed a new feb firmware
+        reconfig febl
+        reconfig febr
+        # will reboot controller
+        reconfig fw0
+
+.. note :: 
+
+    If the detector servers did not start up automatically after reboot, you need to add scripts to do that. See :ref:`Automatic start<Automatic start servers>` for more details.
+
 Jungfrau
 -------------
 
@@ -94,75 +89,37 @@ Download
 - `pof files <https://github.com/slsdetectorgroup/slsDetectorFirmware>`__
 
 
-Upgrade (from v4.x.x)
-^^^^^^^^^^^^^^^^^^^^^^
+Upgrade
+^^^^^^^^
+
+.. warning ::
+
+    In case you have had issues in the past with programming via software:
+
+    * 6.1.2 server has a fix for seamless fpga programming
+
+    * We recommend first updating the on-board detector server to 6.1.2 (with client 6.1.x) using command 'updatedetectorserver' or 'copydetectorserver'.
+
+    * Then use command 'programfpga' to only update firmware or use command 'update' to update firmware and server to the latest release.
+
+
 
 Check :ref:`firmware troubleshooting <blackfin firmware troubleshooting>` if you run into issues while programming firmware.
 
-#. Tftp must be installed on pc.
 
-#. Update client package to the latest (5.x.x).
 
-#. Disable server respawning or kill old server
+
+Program from console
     .. code-block:: bash
 
-        # Option 1: if respawning enabled
-        telnet bchipxxx
-        # edit /etc/inittab
-        # comment out line #ttyS0::respawn:/jungfrauDetectorServervxxx
-        reboot
-        # ensure servers did not start up after reboot
-        telnet bchipxxx
-        ps
+        # These instructions are for upgrades from v5.0.0. For earlier versions, please contact us.
 
-        #  Option 2: if respawning already disabled
-        telnet bchipxxx
-        killall jungfrauDetectorServerv*
+        # Always ensure that the client and server software are of the same release.
 
-#. Copy new server and start in update mode
-    .. code-block:: bash
-
-        tftp pcxxx -r jungfrauDetectorServervxxx -g
-        chmod 777 jungfrauDetectorServervxxx
-        ./jungfrauDetectorServervxxx -u
-
-#. Program fpga from the client console
-    .. code-block:: bash
-
-        sls_detector_get free
-        # Crucial that the next command executes without any errors
-        sls_detector_put hostname bchipxxx
-        sls_detector_put programfpga xxx.pof
-
-#. After programming, kill 'update server' using Ctrl + C in server console.
-
-#. Enable server respawning if needed
-    .. code-block:: bash
-
-        telnet bchipxxx
-        # edit /etc/inittab
-        # uncomment out line #ttyS0::respawn:/jungfrauDetectorServervxxx
-        # ensure the line has the new server name
-        reboot
-        # ensure both servers are running using ps
-        jungfrauDetectorServervxxx
-        jungfrauDetectorServervxxx --stop-server 1953
-
-
-Upgrade (from v5.0.0)
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Check :ref:`firmware troubleshooting <blackfin firmware troubleshooting>` if you run into issues while programming firmware.
-
-Always ensure that the client and server software are of the same release.
-
-
-#. Program from console
-    .. code-block:: bash
-
-        # copies server from tftp folder of pc, programs fpga,
-        # removes old server from respawn, sets up new server to respawn
-        # and reboots
+        # copies server from tftp folder of pc, links new server to jungfrauDetectorServer, 
+        # removes old server from respawn, sets up new lnked server to respawn
+        # programs fpga,
+        # reboots
         sls_detector_put update jungfrauDetectorServervxxx pcxxx xx.pof
 
         # Or only program firmware
@@ -170,8 +127,8 @@ Always ensure that the client and server software are of the same release.
 
 
 
-Gotthard
----------
+Gotthard I
+-----------
 
 Download 
 ^^^^^^^^^^^^^
@@ -186,7 +143,7 @@ Upgrade
 ^^^^^^^^
 .. warning ::
     | Gotthard firmware cannot be upgraded remotely and requires the use of USB-Blaster.
-    | It is generally updated by the SLS Detector group.
+    | It is generally updated by us.
 
 #. Download `Altera Quartus software or Quartus programmer <https://fpgasoftware.intel.com/20.1/?edition=standard&platform=linux&product=qprogrammer#tabs-4>`__.
    
@@ -197,7 +154,7 @@ Upgrade
 
 #. Plug the end of your USB-Blaster with the adaptor provided to the connector 'AS config' on the Gotthard board.
 
-#. Click on 'Add file'. Select programming (pof) file provided by the SLS Detector group.
+#. Click on 'Add file'. Select programming (pof) file provided by us.
 
 #. Check "Program/Configure" and "Verify". Push the start button. Wait until the programming process is finished.
 
@@ -206,68 +163,65 @@ Upgrade
 #. Reboot the detector.
 
 
-Mythen3
--------
-
-.. note :: 
-
-  As it is still in developement, the rbf files must be picked up from the SLS Detector Group.
+Mythen III
+-----------
 
 Download 
 ^^^^^^^^^^^^^
 
 - detector server corresponding to package in slsDetectorPackage/serverBin
 
-- rbf files (in developement)
+- `rbf files <https://github.com/slsdetectorgroup/slsDetectorFirmware>`__
 
 
-Upgrade (from v5.0.0)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Upgrade
+^^^^^^^^
 
-Always ensure that the client and server software are of the same release.
-
-#. Program from console
+Program from console
     .. code-block:: bash
 
-        # copies server from tftp folder of pc, programs fpga,
-        # and reboots (new server not respawned currently)
+        # Always ensure that the client and server software are of the same release.
+
+        # copies server from tftp folder of pc, links new server to mythen3DetectorServer, 
+        # programs fpga,
+        # reboots
         sls_detector_put update mythen3DetectorServervxxx pcxxx xxx.rbf
 
         # Or only program firmware
         sls_detector_put programfpga xxx.rbf
 
-
-
-Gotthard2
--------------
-
 .. note :: 
 
-  As it is still in developement, the rbf files must be picked up from the SLS Detector Group.
+    If the detector servers did not start up automatically after reboot, you need to add scripts to do that. See :ref:`Automatic start<Automatic start servers>` for more details.
+
+Gotthard II
+-------------
 
 Download 
 ^^^^^^^^^^^^^
 - detector server corresponding to package in slsDetectorPackage/serverBin
 
-- rbf files (in development)
+- `rbf files <https://github.com/slsdetectorgroup/slsDetectorFirmware>`__
 
+Upgrade
+^^^^^^^^
 
-Upgrade (from v5.0.0)
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Always ensure that the client and server software are of the same release.
-
-#. Program from console
+Program from console
     .. code-block:: bash
 
-        # copies server from tftp folder of pc, programs fpga,
-        # and reboots (new server not respawned currently)
+        # Always ensure that the client and server software are of the same release.
+        
+        # copies server from tftp folder of pc, links new server to gotthard2DetectorServer, 
+        # programs fpga,
+        # reboots
         sls_detector_put update gotthard2DetectorServervxxx pcxxx xxx.rbf
 
         # Or only program firmware
         sls_detector_put programfpga xxx.rbf
 
+.. note :: 
 
+    If the detector servers did not start up automatically after reboot, you need to add scripts to do that. See :ref:`Automatic start<Automatic start servers>` for more details.
 
 Moench
 -------
@@ -279,19 +233,34 @@ Download
 - `pof files <https://github.com/slsdetectorgroup/slsDetectorFirmware>`__
 
 
-Upgrade (from v5.0.0)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Upgrade
+^^^^^^^^
+
+.. warning ::
+
+    In case you have had issues in the past with programming via software:
+
+    * 6.1.2 server has a fix for seamless fpga programming
+
+    * We recommend first updating the on-board detector server to 6.1.2 (with client 6.1.x) using command 'updatedetectorserver' or 'copydetectorserver'.
+
+    * Then use command 'programfpga' to only update firmware or use command 'update' to update firmware and server to the latest release.
+
+
 
 Check :ref:`firmware troubleshooting <blackfin firmware troubleshooting>` if you run into issues while programming firmware.
 
-Always ensure that the client and server software are of the same release.
 
-#. Program from console
+Program from console
     .. code-block:: bash
 
-        # copies server from tftp folder of pc, programs fpga,
-        # removes old server from respawn, sets up new server to respawn
-        # and reboots
+        # Always ensure that the client and server software are of the same release.
+
+        # copies server from tftp folder of pc, links new server to moenchDetectorServer, 
+        # removes old server from respawn, sets up new lnked server to respawn
+        # programs fpga,
+        # reboots
         sls_detector_put update moenchDetectorServervxxx pcxxx xx.pof
 
         # Or only program firmware
@@ -307,19 +276,22 @@ Download
 - `pof files <https://github.com/slsdetectorgroup/slsDetectorFirmware>`__
 
 
-Upgrade (from v5.0.0)
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Upgrade
+^^^^^^^^
 
 Check :ref:`firmware troubleshooting <blackfin firmware troubleshooting>` if you run into issues while programming firmware.
 
-Always ensure that the client and server software are of the same release.
 
-#. Program from console
+Program from console
     .. code-block:: bash
 
-        # copies server from tftp folder of pc, programs fpga,
-        # removes old server from respawn, sets up new server to respawn
-        # and reboots
+        # Always ensure that the client and server software are of the same release.
+        
+        # copies server from tftp folder of pc, links new server to ctbDetectorServer, 
+        # removes old server from respawn, sets up new lnked server to respawn
+        # programs fpga,
+        # reboots
         sls_detector_put update ctbDetectorServervxxx pcxxx xx.pof
 
         # Or only program firmware
@@ -354,43 +326,53 @@ Firmware Troubleshooting with blackfin
 
 5. If one can't list it, read the next section to try to get the blackfin to list it.
 
-How to get back mtd3 drive remotely (copying new kernel)
+How to get back mtd3 drive remotely (udpating kernel)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    You have 2 alternatives to update the kernel.
 
-  .. code-block:: bash
-    
-    # step 1: get the kernel image (uImage.lzma) from slsdetectorgroup
-    # and copy it to pc's tftp folder
+    1. Commands via software (>= v6.0.0)
 
-    # step 2: connect to the board
-    telnet bchipxxx
+        .. code-block:: bash
 
-    #step 3: go to directory for space
-    cd /var/tmp/
+            sls_detector_put updatekernel /home/...path-to-kernel-image
 
-    # step 3: copy kernel to board
-    tftp pcxxx -r uImage.lzma -g
 
-    # step 4: verify kernel copied properly
-    ls -lrt
-    
-    # step 5: erase flash
-    flash_eraseall /dev/mtd1
-    
-    # step 6: copy new image to kernel drive
-    cat uImage.lzma > /dev/mtd1
-    
-    # step 7:
-    sync
-    
-    # step 8:
-    reboot
-    
-    # step 9: verification
-    telnet bchipxxx
-    uname -a # verify kernel date
-    more /proc/mtd # verify mtd3 is listed
+    2. or command line 
     
+        .. code-block:: bash
+            
+            # step 1: get the kernel image (uImage.lzma) from slsdetectorgroup
+            # and copy it to pc's tftp folder
+
+            # step 2: connect to the board
+            telnet bchipxxx
+
+            #step 3: go to directory for space
+            cd /var/tmp/
+
+            # step 3: copy kernel to board
+            tftp pcxxx -r uImage.lzma -g
+
+            # step 4: verify kernel copied properly
+            ls -lrt
+            
+            # step 5: erase flash
+            flash_eraseall /dev/mtd1
+            
+            # step 6: copy new image to kernel drive
+            cat uImage.lzma > /dev/mtd1
+            
+            # step 7:
+            sync
+            
+            # step 8:
+            reboot
+            
+            # step 9: verification
+            telnet bchipxxx
+            uname -a # verify kernel date
+            more /proc/mtd # verify mtd3 is listed
+            
 
 Last Resort using USB Blaster
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

docs/src/gendoc.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 /**
  * Utility program to generate input files for the command line
  * documentation. Uses the string returned from sls_detector_help cmd
@@ -43,6 +45,7 @@ int main() {
 
     for (const auto &cmd : commands) {
         std::ostringstream os;
+        std::cout << cmd << '\n';
         proxy.Call(cmd, {}, -1, slsDetectorDefs::HELP_ACTION, os);
 
         auto tmp = os.str().erase(0, cmd.size());

docs/src/index.rst

@@ -66,6 +66,15 @@ Welcome to slsDetectorPackage's documentation!
     virtualserver
     serverdefaults
 
+
+.. toctree::
+    :caption: Detector UDP Header
+    :maxdepth: 2
+
+    udpheader
+    udpconfig
+    udpdetspec
+
 .. toctree::
     :caption: Receiver
     :maxdepth: 2

docs/src/installation.rst

@@ -13,88 +13,27 @@
     
     Before building from source make sure that you have the 
     :doc:`dependencies <../dependencies>` installed. If installing using conda, conda will 
-    manage the dependencies.
-    
+    manage the dependencies. Avoid also installing packages with pip. 
+   
 
 .. _Installation:
 
+
 Installation
-==============================================
-
-.. _build from source using cmake:
-
-Build from source using CMake
----------------------------------
-
-Note that on some systems, for example RH7,  cmake v3+ is available under the cmake3 alias.
-It is also required to clone with the option --recursive to get the git submodules used
-in the package. 
-
-
-.. code-block:: bash
-
-    git clone --recursive https://github.com/slsdetectorgroup/slsDetectorPackage.git
-    mkdir build && cd build
-    cmake ../slsDetectorPackage -DCMAKE_INSTALL_PREFIX=/your/install/path
-    make -j12 #or whatever number of cores you are using to build
-    make install
-
-The easiest way to configure options is to use the ccmake utility. 
-
-.. code-block:: bash
-
-    #from the build directory
-    ccmake .
-
-
-Build using cmk.sh script
--------------------------
-These are mainly aimed at those not familiar with using ccmake and cmake.
-
-.. code-block:: bash
-
-    The binaries are generated in slsDetectorPackage/build/bin directory.
-
-    Usage: $0 [-c] [-b] [-p] [e] [t] [r] [g] [s] [u] [i] [m] [n] [-h] [z] [-d <HDF5 directory>] [-l Install directory] [-k <CMake command>] [-j <Number of threads>]
-    -[no option]: only make
-    -c: Clean
-    -b: Builds/Rebuilds CMake files normal mode
-    -p: Builds/Rebuilds Python API
-    -h: Builds/Rebuilds Cmake files with HDF5 package
-    -d: HDF5 Custom Directory
-    -k: CMake command
-    -l: Install directory
-    -t: Build/Rebuilds only text client
-    -r: Build/Rebuilds only receiver
-    -g: Build/Rebuilds only gui
-    -s: Simulator
-    -u: Chip Test Gui
-    -j: Number of threads to compile through
-    -e: Debug mode
-    -i: Builds tests
-    -m: Manuals
-    -n: Manuals without compiling doxygen (only rst)
-    -z: Moench zmq processor
-    
-    # get all options
-    ./cmk.sh -?
-
-    # new build  and compile in parallel:
-    ./cmk.sh -bj5
-
+===============
 
 Install binaries using conda
---------------------------------
+-------------------------------
 
 Conda is not only useful to manage python environments but can also
-be used as a user space package manager. 
+be used as a user space package manager. Dates in the tag (for eg. 2020.07.23.dev0) 
+are from the developer branch. Please use released tags for stability.
 
 We have three different packages available:
 
- * **slsdetlib**, shared libraries and command line utilities 
- * **slsdetgui**, GUI
- * **slsdet**, Python bindings
-
+    * **slsdetlib** shared libraries and command line utilities 
+    * **slsdetgui** GUI
+    * **slsdet** Python bindings
 
 .. code-block:: bash
 
@@ -103,9 +42,9 @@ We have three different packages available:
     conda config --add channels slsdetectorgroup
     conda config --set channel_priority strict
 
-    #cerate an environment with our library, then activate
-    #replace 2020.07.20.dev0 with the required tag
-    conda create -n myenv slsdetlib=2020.07.23.dev0
+    #create and activate an environment with our library
+    #replace 6.1.1 with the required tag
+    conda create -n myenv slsdetlib=6.1.1
     conda activate myenv
 
     #ready to use
@@ -116,11 +55,145 @@ We have three different packages available:
 .. code-block:: bash
 
     #List available versions
+    # lib and binaries
+    conda search slsdetlib
+    # python
     conda search slsdet
+    # gui
+    conda search slsdetgui
 
 
-Build from source on old distributions
------------------------------------------
+
+
+Build from source
+-------------------
+
+
+1. Download Source Code from github
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: bash
+
+    git clone https://github.com/slsdetectorgroup/slsDetectorPackage.git --branch 6.1.1
+
+
+| **Pybind for Python**
+| v7.0.0+:
+|   pybind11 packaged into 'libs/pybind'. No longer a submodule. No need for "recursive" or "submodule update".
+| 
+| Older versions:
+|   pybind11 is a submodule. Must be cloned using "recursive" and updated when switching between versions using the following commands.
+
+.. code-block:: bash
+    
+    # clone using recursive to get pybind11 submodule
+    git clone --recursive https://github.com/slsdetectorgroup/slsDetectorPackage.git
+
+    # update submodule when switching between releases
+    cd slsDetectorPackage
+    git submodule update --init
+   
+
+.. _build from source using cmake:
+
+
+
+2. Build from Source
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Build using CMake
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: bash
+
+    # outside slsDetecorPackage folder
+    mkdir build && cd build
+
+    # configure & generate Makefiles using cmake
+    # by listing all your options (alternately use ccmake described below)
+    # cmake3 for some systems
+    cmake ../slsDetectorPackage -DCMAKE_INSTALL_PREFIX=/your/install/path
+
+    # compiled to the build/bin directory
+    make -j12 #or whatever number of cores you are using to build
+
+    # install headers and libs in /your/install/path directory
+    make install
+
+
+Instead of the cmake command, one can use ccmake to get a list of options to configure and generate Makefiles at ease.
+
+.. code-block:: bash
+
+    # ccmake3 for some systems
+    ccmake ..
+ 
+    # choose the options
+    # first press [c] - configure
+    # then press [g] - generate
+
+
+
+===============================     ===========================================
+Example cmake options               Comment
+===============================     ===========================================
+-DSLS_USE_PYTHON=ON                 Python
+-DPython_FIND_VIRTUALENV=ONLY       Python from only the conda environment 
+-DZeroMQ_HINT=/usr/lib64            Use system zmq instead
+-DSLS_USE_GUI=ON                    GUI
+===============================     ===========================================
+
+
+
+Build using in-built cmk.sh script
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+.. code-block:: bash
+
+    The binaries are generated in slsDetectorPackage/build/bin directory.
+
+    Usage: ./cmk.sh [-b] [-c] [-d <HDF5 directory>] [e] [g] [-h] [i] [-j <Number of threads>] 
+    [-k <CMake command>] [-l <Install directory>] [m] [n] [-p] [-q <Zmq hint directory>] 
+    [r] [s] [t] [u] [z]  
+    -[no option]: only make
+    -b: Builds/Rebuilds CMake files normal mode
+    -c: Clean
+    -d: HDF5 Custom Directory
+    -e: Debug mode
+    -g: Build/Rebuilds only gui
+    -h: Builds/Rebuilds Cmake files with HDF5 package
+    -i: Builds tests
+    -j: Number of threads to compile through
+    -k: CMake command
+    -l: Install directory
+    -m: Manuals
+    -n: Manuals without compiling doxygen (only rst)
+    -p: Builds/Rebuilds Python API
+    -q: Zmq hint directory
+    -r: Build/Rebuilds only receiver
+    -s: Simulator
+    -t: Build/Rebuilds only text client
+    -u: Chip Test Gui
+    -z: Moench zmq processor
+
+    
+    # get all options
+    ./cmk.sh -?
+
+    # new build and compile in parallel:
+    ./cmk.sh -bj5
+
+    # new build, python and compile in parallel:
+    ./cmk.sh -bpj5
+
+    #To use the system zmq (/usr/lib64) instead
+    ./cmk.sh -bj5 -q /usr/lib64
+
+
+
+Build on old distributions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 If your linux distribution doesn't come with a C++11 compiler (gcc>4.8) then 
 it's possible to install a newer gcc using conda and build the slsDetectorPackage
@@ -131,12 +204,64 @@ using this compiler
     #Create an environment with the dependencies
     conda create -n myenv gxx_linux-64 cmake zmq
     conda activate myenv
+
+    # outside slsDetecorPackage folder
+    mkdir build && cd build
     cmake ../slsDetectorPackage -DCMAKE_PREFIX_PATH=$CONDA_PREFIX
     make -j12
 
 
+
+Build slsDetectorGui (Qt5)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+1. Using pre-built binary on conda
+    .. code-block:: bash
+
+        conda create -n myenv slsdetgui=7.0.0
+        conda activate myenv
+
+
+2. Using system installation on RHEL7
+    .. code-block:: bash
+
+        yum install qt5-qtbase-devel.x86_64
+        yum install qt5-qtsvg-devel.x86_64 
+
+
+3. Using conda
+    .. code-block:: bash
+
+        #Add channels for dependencies and our library
+        conda config --add channels conda-forge
+        conda config --add channels slsdetectorgroup
+        conda config --set channel_priority strict
+
+        # create environment to compile
+        # on rhel7
+        conda create -n slsgui zeromq gxx_linux-64 gxx_linux-64 mesa-libgl-devel-cos6-x86_64 qt
+        # on fedora or newer systems
+        conda create -n slsgui zeromq qt
+
+        # when using conda compilers, would also need libgl, but no need for it on fedora unless maybe using it with ROOT
+
+        # activate environment
+        conda activate slsgui
+
+        # compile with cmake outside slsDetecorPackage folder
+        mkdir build && cd build
+        cmake ../slsDetectorPackage -DSLS_USE_GUI=ON
+        make -j12
+
+        # or compile with cmk.sh
+        cd slsDetectorPackage
+        ./cmk.sh -cbgj9
+
+
+
+
 Build this documentation
--------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^
 
 The documentation for the slsDetectorPackage is build using a combination 
 of Doxygen, Sphinx and Breathe. The easiest way to install the dependencies
@@ -144,11 +269,15 @@ is to use conda
 
 .. code-block:: bash
 
-    conda create -n myenv python sphinx sphinx_rtd_theme
+    conda create -n myenv python sphinx_rtd_theme breathe
 
-Then enable the option SLS_BUILD_DOCS to create the targets
 
 .. code-block:: bash
 
+    # using cmake or ccmake to enable DSLS_BUILD_DOCS
+    # outside slsDetecorPackage folder
+    mkdir build && cd build
+    cmake ../slsDetectorPackage -DSLS_BUILD_DOCS=ON
+
     make docs # generate API docs and build Sphinx RST
-    make rst # rst only, saves time in case the API did not change
+    make rst # rst only, saves time in case the API did not change

docs/src/pyenums.rst

@@ -71,4 +71,19 @@ exposed to Python through pybind11.
     :undoc-members:
 
 .. autoclass:: timingSourceType
+    :undoc-members:
+
+.. autoclass:: M3_GainCaps
+    :undoc-members:    
+
+.. autoclass:: portPosition
+    :undoc-members:
+
+.. autoclass:: streamingInterface
+    :undoc-members:
+
+.. autoclass:: vetoAlgorithm
+    :undoc-members:
+
+.. autoclass:: gainMode
     :undoc-members:

docs/src/pyexamples.rst

@@ -10,27 +10,46 @@ open an issue in our our  `github repo
 Setting exposure time 
 ------------------------------------
 
-Setting and reading back exposure time can be done either using a Python datetime.timedelta
-or by setting the time in seconds. 
+Setting and reading back exposure time can be done either using a Python 
+datetime.timedelta, DurationWrapper or by setting the time in seconds. 
 
 ::
 
     # Set exposure time to 1.2 seconds
     >>> d.exptime = 1.2
+    >>> d.exptime = 5e-07
 
-    # Setting exposure time using timedelta
+    # Setting exposure time using timedelta (upto microseconds precision)
     import datetime as dt
     >>> d.exptime = dt.timedelta(seconds = 1.2)
+    >>> d.exptime = dt.timedelta(seconds = 1, microseconds = 3)
 
     # With timedelta any arbitrary combination of units can be used
     >>> t = dt.timedelta(microseconds = 100, seconds = 5.3, minutes = .3)
 
+    # using DurationWrapper to set in seconds
+    >>> from slsdet import DurationWrapper
+    >>> d.exptime = DurationWrapper(1.2)
+    
+    # using DurationWrapper to set in ns
+    >>> t = DurationWrapper()
+    >>> t.set_count(500)
+    >>> d.exptime = t
+
     # To set exposure time for individual detector one have to resort
     # to the C++ style API.
     # Sets exposure time to 1.2 seconds for module 0, 6 and 12
     >>> d.setExptime(1.2, [0, 6, 12]) 
     >>> d.setExptime(dt.timedelta(seconds = 1.2), [0, 6, 12]) 
 
+    # to get in seconds
+    >>> d.period
+    181.23
+
+    # to get in DurationWrapper
+    >>> d.getExptime()
+    [sls::DurationWrapper(total_seconds: 181.23 count: 181230000000)]
+
 
 
 ------------------------------------
@@ -220,8 +239,7 @@ Setting and getting times
     # This sets the exposure time for all modules
     d.exptime = 0.5
 
-    # exptime also accepts a python datetime.timedelta
-    # which can be used to set the time in almost any unit
+    # exptime also accepts a python datetime.timedelta (upto microseconds resolution)
     t = dt.timedelta(milliseconds = 2.3)
     d.exptime = t
 
@@ -229,16 +247,25 @@ Setting and getting times
     t = dt.timedelta(minutes = 3, seconds = 1.23)
     d.exptime = t
 
+    # using DurationWrapper to set in seconds
+    >>> from slsdet import DurationWrapper
+    >>> d.exptime = DurationWrapper(1.2)
+    
+    # using DurationWrapper to set in ns
+    >>> t = DurationWrapper()
+    >>> t.set_count(500)
+    >>> d.exptime = t
+
     # exptime however always returns the time in seconds
     >>> d.exptime
     181.23 
 
     # To get back the exposure time for each module 
     # it's possible to use getExptime, this also returns
-    # the values as datetime.timedelta
+    # the values as  DurationWrapper
 
     >>> d.getExptime()
-    [datetime.timedelta(seconds=181, microseconds=230000), datetime.timedelta(seconds=181, microseconds=230000)]
+    [sls::DurationWrapper(total_seconds: 181.23 count: 181230000000)]
 
     # In case the values are the same it's possible to use the
     # element_if_equal function to reduce the values to a single 
@@ -246,7 +273,8 @@ Setting and getting times
 
     >>> t = d.getExptime()
     >>> element_if_equal(t)
-    datetime.timedelta(seconds=1)
+    sls::DurationWrapper(total_seconds: 1.2 count: 1200000000)
+
 
 --------------
 Reading dacs

docs/src/pygettingstarted.rst

@@ -6,7 +6,7 @@ Getting Started
 Which Python?  
 --------------------
 
-We require at lest Python 3.6 and strongly recommended that you don't use the system
+We require at least Python 3.6 and strongly recommended that you don't use the system
 Python installation. The examples in this documentation uses `conda
 <https://docs.conda.io/en/latest/miniconda.html>`_ since it provides good support
 also for non Python packages but there are also other alternatives like, pyenv. 
@@ -14,28 +14,35 @@ also for non Python packages but there are also other alternatives like, pyenv.
 Using something like conda also allows you to quickly switch beteen different Python 
 environments. 
 
-.. warning ::
+---------------------
+Building from Source 
+---------------------
+
+If you are not installing slsdet binaries from conda, but instead building from 
+source, please refer to  :ref:`the installation section<Installation>` for details.
+
+Don't forget to compile with the option SLS_USE_PYTHON=ON to enable the Python 
+bindings or if you use the cmk.sh script -p.
+
+.. note ::
+
+    Ensure that the sls det python lib compiled is for the expected python version.
+    For example, build/bin/_slsdet.cpython-39-x86_64-linux-gnu.so for Python v3.9.x
 
-    If you use conda avoid also installing packages with pip. 
 
 ---------------------
 PYTHONPATH 
 ---------------------
 
-If you install slsdet using conda everything is set up and you can
+If you install slsdet binaries using conda everything is set up and you can
 directly start using the Python bindings. However, if you build 
-from source you need to tell Python where to find slsdet. This
-is be done by adding your build/bin directory to PYTHONPATH. 
+from source you need to tell Python where to find slsdet to use it. This
+can be done by adding your build/bin directory to PYTHONPATH. 
 
 .. code-block:: bash
 
     export PYTHONPATH = /path/to/your/build/bin:$PYTHONPATH
 
-.. note ::
-
-    Don't forget to compile with the option SLS_USE_PYTHON=ON to enable
-    the Python bindings or if you use the cmk.sh script -p.
-
 --------------------------------------
 Which detector class should I use? 
 --------------------------------------
@@ -136,7 +143,7 @@ can use dir()
     '__str__', '__subclasshook__', '_adc_register', '_frozen', 
     '_register', 'acquire', 'adcclk', 'adcphase', 'adcpipeline', 
     'adcreg', 'asamples', 'auto_comp_disable', 'clearAcquiringFlag', 
-    'clearBit', 'clearROI', 'client_version', 'config', 'copyDetectorServer', 
+    'clearBit', 'clearROI', 'client_version', 'config',  
     'counters', 'daclist', 'dacvalues', 'dbitclk', 'dbitphase' ...
 
 Since the list for Detector is rather long it's an good idea to filter it. 
@@ -146,9 +153,11 @@ their name.
 :: 
 
     >>> [item for item in dir(d) if 'time' in item]
-    ['exptime', 'getExptime', 'getExptimeForAllGates', 'getExptimeLeft', 
-    'getSubExptime', 'patwaittime0', 'patwaittime1', 'patwaittime2', 
-    'setExptime', 'setSubExptime', 'subdeadtime', 'subexptime']
+    ['compdisabletime', 'exptime', 'exptimel', 'frametime', 'getExptime', 
+    'getExptimeForAllGates', 'getExptimeLeft', 'getSubExptime', 'patwaittime', 
+    'patwaittime0', 'patwaittime1', 'patwaittime2', 'runtime', 'setExptime', 
+    'setSubExptime', 'subdeadtime', 'subexptime']
+
 
 The above method works on any Python object but for convenience we also 
 included two functions to find names. View prints the names one per line
@@ -160,6 +169,7 @@ while find returns a list of names.
 
     >>> view('exptime')
     exptime
+    exptimel
     getExptime
     getExptimeForAllGates
     getExptimeLeft
@@ -168,6 +178,7 @@ while find returns a list of names.
     setSubExptime
     subexptime
 
+
     >>> find('exptime')
     ['exptime', 'getExptime', 'getExptimeForAllGates', 'getExptimeLeft', 
     'getSubExptime', 'setExptime', 'setSubExptime', 'subexptime']
@@ -185,19 +196,39 @@ To access the documentation of a function directly from the Python prompt use he
     Help on property:
 
         Period between frames, accepts either a value in seconds or datetime.timedelta
-
+        
         Note
         -----
-        :getter: always returns in seconds. To get in datetime.delta, use getPeriod
-
-        Examples
+        :getter: always returns in seconds. To get in DurationWrapper, use getPeriod
+        
+        Example
         -----------
+        >>> # setting directly in seconds
         >>> d.period = 1.05
-        >>> d.period = datetime.timedelta(minutes = 3, seconds = 1.23)
+        >>>
+        >>> # setting directly in seconds
+        >>> d.period = 5e-07
+        >>> 
+        >>> # using timedelta (up to microseconds precision)
+        >>> from datatime import timedelta
+        >>> d.period = timedelta(seconds = 1, microseconds = 3)
+        >>> 
+        >>> # using DurationWrapper to set in seconds
+        >>> from slsdet import DurationWrapper
+        >>> d.period = DurationWrapper(1.2)
+        >>> 
+        >>> # using DurationWrapper to set in ns
+        >>> t = DurationWrapper()
+        >>> t.set_count(500)
+        >>> d.period = t
+        >>>
+        >>> # to get in seconds
         >>> d.period
         181.23
-        >>> d.getPeriod()
-        [datetime.timedelta(seconds=181, microseconds=230000)]
+        >>> 
+        >>> d.getExptime()
+        [sls::DurationWrapper(total_seconds: 181.23 count: 181230000000)]
+
 
 
 ----------------------
@@ -217,11 +248,12 @@ The enums can be found in slsdet.enums
 
     import slsdet
     >>> [e for e in dir(slsdet.enums) if not e.startswith('_')]
-    ['burstMode', 'clockIndex', 'dacIndex', 
-    'detectorSettings', 'detectorType', 'dimension', 'externalSignalFlag', 
-    'fileFormat', 'frameDiscardPolicy', 
-    'readoutMode', 'runStatus', 'speedLevel', 'timingMode', 
-    'timingSourceType']
+    ['M3_GainCaps', 'burstMode', 'clockIndex', 'cls', 'dacIndex', 'detectorSettings', 
+    'detectorType', 'dimension', 'externalSignalFlag', 'fileFormat', 
+    'frameDiscardPolicy', 'gainMode', 'name', 'polarity', 'portPosition', 
+    'readoutMode', 'runStatus', 'speedLevel', 'streamingInterface', 'timingMode', 
+    'timingSourceType', 'vetoAlgorithm']
+
 
     # Even though importing using * is not recommended one could
     # get all the enums like this: 

docs/src/quick_start_guide.rst

@@ -66,11 +66,15 @@ For a Single Module
         # sets destination udp ports (not needed, default is 50001)
         udp_dstport 50012
 
+        # 1g data out
         # source udp ips must be same subnet at destintaion udp ips
-        udp_srcip 192.168.1.112
-
+        # udp_srcip 192.168.1.112
         # destination udp ip picked up from rx_hostname (if auto)
-        udp_dstip auto
+        # udp_dstip auto
+
+        # 10g data out
+        udp_srcip 10.30.20.200
+        udp_dstip 10.30.20.6
 
         # set file path
         fpath /tmp

docs/src/receivers.rst

@@ -4,41 +4,8 @@ Receivers
 Receiver processes can be run on same or different machines as the client, receives the data from the detector (via UDP packets).
 When using the slsReceiver/ slsMultiReceiver, they can be further configured by the client control software (via TCP/IP) to set file name, file path, progress of acquisition etc.
 
-Detector UDP Header
----------------------
 
-| The UDP data format for the packets consist of a common header for all detectors, followed by the data for that one packet.
-
-**The SLS Detector Header**
-
-.. table:: <-------------------------------- 8 bytes -------------------------------->
-    :align: center
-    :widths: 30,30,30,30
-
-    +--------------------------------------------------------------------+
-    |frameNumber                                                         |
-    +---------------------------------+----------------------------------+
-    |expLength                        |packetNumber                      |
-    +---------------------------------+----------------------------------+
-    |bunchId                                                             |
-    +--------------------------------------------------------------------+
-    |timestamp                                                           |
-    +----------------+----------------+----------------+-----------------+
-    |modId           |row             |column          |reserved         |
-    +----------------+----------------+----------------+--------+--------+
-    |debug                            |roundRNumber    |detType |version |
-    +---------------------------------+----------------+--------+--------+
-
-UDP configuration in Config file
-----------------------------------
-
-#. UDP source port is hardcoded in detector server, starting at 32410.
-#. **udp_dstport** : UDP destination port number. Port in receiver pc to listen to packets from the detector.
-#. **udp_dstip** : IP address of UDP destination interface. IP address of interface in receiver pc to listen to packets from detector. If **auto** is used (only when using slsReceiver/ slsMultiReceiver), the IP of **rx_hostname** is picked up.
-#. **udp_dstmac** : Mac address of UDP destination interface. MAC address of interface in receiver pc to list to packets from detector. Only required when using custom receiver, else slsReceiver/slsMultiReceiver picks it up from **udp_dstip**.
-#. **udp_srcip** : IP address of UDP source interface. IP address of detector UDP interface to send packets from. Do not use for Eiger 1Gb interface (uses its hardware IP). For others, must be in the same subnet as **udp_dstip**.
-#. **udp_srcmac** : MAC address of UDP source interface. MAC address of detector UDP interface to send packets from. Do not use for Eiger (uses hardware mac). For others, it is not necessary, but can help for switch and debugging to put unique values for each module.
- 
+To know more about detector receiver configuration, please check out :ref:`detector udp header and udp commands in the config file <detector udp header>`
 
 Custom Receiver
 ----------------

docs/src/servers.rst

@@ -1,5 +1,14 @@
-Detector Servers
-=================
+Getting Started
+===============
+
+Detector Servers include:
+   * Control server [default port: 1952]
+      * Almost all client communication.
+   * Stop server [default port: 1953]
+      *  Client requests for detector status, stop acquisition, temperature, advanced read/write registers.
+
+When using a blocking acquire command (sls_detector_acquire or Detector::acquire), the control server is blocked until end of acquisition. However, stop server commands could be used in parallel.
+
 
 Location
 ---------
@@ -14,28 +23,22 @@ Arguments
    .. code-block:: bash  
 
       Possible arguments are:
-      -v, --version            : Software version
-      -p, --port <port>        : TCP communication port with client. 
-      -g, --nomodule           : [Mythen3][Gotthard2] Generic or No Module mode. 
-                                 Skips detector type checks.
-      -f, --phaseshift <value> : [Gotthard] only. Sets phase shift. 
-      -d, --devel              : Developer mode. Skips firmware checks. 
-      -u, --update             : Update mode. Skips firmware checks and initial detector setup. 
-      -s, --stopserver         : Stop server. Do not use as it is created by control server 
-
-
-Basics
-------------
-
-Detector Servers include:
-   * Control server [default port: 1952]
-      * Almost all client communication.
-   * Stop server [default port: 1953]
-      *  Client requests for detector status, stop acquisition, temperature, advanced read/write registers.
-
-When using a blocking acquire command (sls_detector_acquire or Detector::acquire), the control server is blocked until end of acquisition. However, stop server commands could be used in parallel.
+         -v, --version            : Software version
+         -p, --port <port>        : TCP communication port with client. 
+         -g, --nomodule           : [Mythen3][Gotthard2] 
+                                    Generic or No Module mode. Skips detector type checks. 
+         -f, --phaseshift <value> : [Gotthard] only. Sets phase shift. 
+         -d, --devel              : Developer mode. Skips firmware checks. 
+         -u, --update             : Update mode. Skips firmware checks and initial detector setup. 
+         -i, --ignore-config      : [Eiger][Jungfrau][Gotthard][Gotthard2][Moench] 
+                                    Ignore config file. 
+         -m, --master <master>    : [Eiger][Mythen3][Gotthard][Gotthard2] 
+                                    Set Master to 0 or 1. Precedence over config file. Only for virtual servers except Eiger. 
+         -t, --top <top>          : [Eiger] Set Top to 0 or 1. Precedence over config file. 
+         -s, --stopserver         : Stop server. Do not use as it is created by control server 
 
 
+.. _Automatic start servers:
 Automatic start 
 ------------------
 
@@ -89,6 +92,8 @@ One can start the on-board detector server automatically upon powering on the bo
       .. code-block:: bash
       
          sync
+
+         # physically reboot for Gotthard II or Mythen III
          reboot
 
          # verify

docs/src/serverupgrade.rst

@@ -1,114 +1,66 @@
-Detector Server Upgrade
-=======================
-
-
-
-Eiger
--------------
+.. _Detector Server Upgrade:
+Upgrade
+========
 
 
 **Location:** slsDetectorPackage/serverBin/ folder for every release.
 
+.. note :: 
+
+    For Mythen3, Gotthard2 and Eiger, you need to add scripts to automatically start detector server upon power on. See :ref:`Automatic start<Automatic start servers>` for more details.
+
+ .. note :: 
+
+    Eiger requires a manual reboot. Or killall the servers and restart the new linked one. If you are in the process of updating firmware, then don't reboot yet.
+
+
+6.1.1+ (no tftp required)
+---------------------------------------
+
+#. Program from console
 
-#. Kill old server and copy new server
     .. code-block:: bash
 
-        # Option 1: from detector console
-        # kill old server
-        ssh root@bebxxx
-        killall eigerDetectorServer
+        # the following command copies new server, creates a soft link to xxxDetectorServerxxx
+        # [Jungfrau][CTB][Moench] also deletes the old server binary and edits initttab to respawn server on reboot
+        # Then, the detector controller will reboot (except Eiger)
+        sls_detector_put updatedetectorserver /complete-path-to-binary/xxxDetectorServerxxx
 
-        # copy new server
-        cd executables
-        scp user@pc:/path/eigerDetectorServerxxx .
-        chmod 777 eigerDetectorServerxxx
-        ln -sf eigerDetectorServerxxx eigerDetectorServer
-        sync
+#. Copy the detector server specific config files or any others required to the detector:
 
-        # Options 2: from client console for multiple modules
-        for i in bebxxx bebyyy;
-        do ssh root@$i killall eigerDetectorServer;
-        scp eigerDetectorServerxxx root@$i:~/executables/eigerDetectorServer;
-        ssh root@$i sync; done
+   .. code-block:: bash
 
+        sls_detector_put execcommand "tftp pcxxx -r configxxx -g"
 
-#. Reboot the detector.
-
-
-Jungfrau
--------------
-
-**Location:** slsDetectorPackage/serverBin/ folder for every release.
+5.0.0 - 6.1.1
+--------------
 
 #. Install tftp and copy detector server binary to tftp folder
-#. Program from console (only from 5.0.0-rcx)
+#. Program from console
+
     .. code-block:: bash
 
-        # copies new server from pc tftp folder, respawns and reboots
-        sls_detector_put copydetectorserver jungfrauDetectorServerxxx pcxxx
+        # the following command copies new server from pc tftp folder, creates a soft link to xxxDetectorServerxxx
+        # [Jungfrau][CTB][Moench] also edits initttab to respawn server on reboot
+        # Then, the detector controller will reboot (except Eiger)
+        sls_detector_put copydetectorserver xxxDetectorServerxxx pcxxx
+
+#. Copy the detector server specific config files or any others required to the detector:
+
+   .. code-block:: bash
+
+        sls_detector_put execcommand "tftp pcxxx -r configxxx -g"
 
 
-Gotthard
----------
+Troubleshooting with tftp
+^^^^^^^^^^^^^^^^^^^^^^^^^
 
-**Location:** slsDetectorPackage/serverBin/ folder for every release.
+#. tftp write error: There is no space left. Please delete some old binaries and try again.
 
-#. Install tftp and copy detector server binary to tftp folder
-#. Program from console (only from 5.0.0-rcx)
-    .. code-block:: bash
-
-        # copies new server from pc tftp folder, respawns and reboots
-        sls_detector_put copydetectorserver gotthardDetectorServerxxx pcxxx
+#. text file busy: You are trying to copy the same server.
 
 
+< 5.0.0
+--------
 
-Mythen3
--------
-
-**Location:** slsDetectorPackage/serverBin/ folder for every release.
-
-#. Install tftp and copy detector server binary to tftp folder
-#. Program from console (only from 5.0.0-rcx)
-    .. code-block:: bash
-
-        # copies new server from pc tftp folder and reboots (does not respawn)
-        sls_detector_put copydetectorserver mythen3DetectorServerxxx pcxxx
-
-
-Gotthard2
-----------
-
-**Location:** slsDetectorPackage/serverBin/ folder for every release.
-
-#. Install tftp and copy detector server binary to tftp folder
-#. Program from console (only from 5.0.0-rcx)
-    .. code-block:: bash
-
-        # copies new server from pc tftp folder and reboots (does not respawn)
-        sls_detector_put copydetectorserver gotthard2DetectorServerxxx pcxxx
-
-
-Moench
-------
-
-**Location:** slsDetectorPackage/serverBin/ folder for every release.
-
-#. Install tftp and copy detector server binary to tftp folder
-#. Program from console (only from 5.0.0-rcx)
-    .. code-block:: bash
-
-        # copies new server from pc tftp folder, respawns and reboots
-        sls_detector_put copydetectorserver moenchDetectorServerxxx pcxxx
-
-
-Ctb
----
-
-**Location:** slsDetectorPackage/serverBin/ folder for every release.
-
-#. Install tftp and copy detector server binary to tftp folder
-#. Program from console (only from 5.0.0-rcx)
-    .. code-block:: bash
-
-        # copies new server from pc tftp folder, respawns and reboots
-        sls_detector_put copydetectorserver ctbDetectorServerxxx pcxxx
+Please contact us.

docs/src/slsreceiver.rst

@@ -86,16 +86,160 @@ Client Commands
         sls_detector_get -h rx_framescaught
 
 
+ZMQ: Json Header Format
+------------------------
+
+
+**Change in field names from slsDetectorPackage v6.x.x to v7.0.0**
+
+* detSpec1 <- bunchId
+* detSpec2 <- reserved
+* detSpec3 <- debug
+* detSpec4 <- roundRNumber
+
+
+**Format**
+
+    .. code-block:: bash 
+
+        {
+            "jsonversion": unsigned int,
+            "bitmode": unsigned int,
+            "fileIndex": unsigned long int,
+            "detshape": [
+                unsigned int,
+                unsigned int
+            ],
+            "shape": [
+                unsigned int,
+                unsigned int
+            ],
+            "size": unsigned int,
+            "acqIndex": unsigned long int,
+            "frameIndex": unsigned long int,
+            "progress": double,
+            "fname": string,
+            "data": unsigned int,
+            "completeImage": unsigned int,
+
+            "frameNumber": unsigned long long int,
+            "expLength": unsigned int,
+            "packetNumber": unsigned int,
+            "detSpec1": unsigned long int,
+            "timestamp": unsigned long int,
+            "modId": unsigned int,
+            "row": unsigned int,
+            "column": unsigned int,
+            "detSpec2": unsigned int,
+            "detSpec3": unsigned int,
+            "detSpec4": unsigned int,
+            "detType": unsigned int,
+            "version": unsigned int,
+            
+            "flipRows": unsigned int,
+            "quad": unsigned int,
+            "addJsonHeader": {
+                string : string
+            }
+        }
+
+   +--------------+----------------------------------------------+
+   |   Field      |       Description                            |
+   +--------------+----------------------------------------------+
+   | jsonversion  | Version of the json header.                  |
+   |              | Value at 4 for v6.x.x and v7.x.x             |
+   +--------------+----------------------------------------------+
+   | bitmode      | Bits per pixel [4|8|16|32]                   |
+   +--------------+----------------------------------------------+
+   | fileIndex    | Current file acquisition index               |
+   +--------------+----------------------------------------------+
+   | detshape     | Geometry of the entire detector              |
+   +--------------+----------------------------------------------+
+   | shape        | Geometry of the current port streamed out    |
+   +--------------+----------------------------------------------+
+   | size         | Size of image of current port in bytesout    |
+   +--------------+----------------------------------------------+
+   | acqIndex     | Frame number from the detector (redundant)   |
+   +--------------+----------------------------------------------+
+   | frameIndex   | Frame number of current acquisition          |
+   |              | (Starting at 0)                              |
+   +--------------+----------------------------------------------+
+   | progress     | Progress of current acquisition in %         |
+   +--------------+----------------------------------------------+
+   | fname        | Current file name                            |
+   +--------------+----------------------------------------------+
+   | data         | 1 if there is data following                 |
+   |              | 0 if dummy header                            |
+   +--------------+----------------------------------------------+
+   | completeImage| 1 if no missing packets for this frame       |
+   |              | in this port, else 0                         |
+   +--------------+----------------------------------------------+
+   | frameNumber  | Frame number                                 |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | expLength    | subframe number (32 bit eiger)               |
+   |              | or real time exposure time in 100ns (others) |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | packetNumber | Number of packets caught for that frame      |
+   +--------------+----------------------------------------------+
+   | detSpec1     | See :ref:`here<Detector Specific Fields>`    |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | timestamp    | Timestamp with 10 MHz clock                  |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | modId        | Module Id                                    |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | row          | Row number in detector                       |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | column       | Column number in detector                    |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | detSpec2     | See :ref:`here<Detector Specific Fields>`    |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | detSpec3     | See :ref:`here<Detector Specific Fields>`    |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | detSpec4     | See :ref:`here<Detector Specific Fields>`    |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | detType      | Detector type enum                           |
+   | detSpec3     | See :ref:`Detector enum<Detector Enum>`      |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | version      | Detector header version. At 2                |
+   |              | [From detector udp header]                   |
+   +--------------+----------------------------------------------+
+   | flipRows     | 1 if rows should be flipped.                 |
+   |              | Usually for Eiger bottom.                    |
+   +--------------+----------------------------------------------+
+   | quad         | 1 if its an Eiger quad.                      |
+   +--------------+----------------------------------------------+
+   | addJsonHeader| Optional custom parameters that is required  |
+   |              | for processing code.                         |
+   +--------------+----------------------------------------------+
+
+
+
 File format
 --------------
 
-* The file name format is [fpath]/[fname]_dx_fy_[findex].raw, where x is module index and y is file index. **fname** is file name prefix and by default "run". **fpath** is '/' by default.
+Master file is in json format.
 
-* Each acquisition will have an increasing acquisition index or findex (if file write enabled). This can be retrieved by using **findex** command. 
+The file name format is [fpath]/[fname]_dx_fy_[findex].raw, where x is module index and y is file index. **fname** is file name prefix and by default "run". **fpath** is '/' by default.
 
-* Each acquisition can have multiple files (the file index number **y**), with **rx_framesperfile** being the maximum number of frames per file. The default varies for each detector type.
 
-* Some file name examples:
+Each acquisition will have an increasing acquisition index or findex (if file write enabled). This can be retrieved by using **findex** command. 
+
+
+Each acquisition can have multiple files (the file index number **y**), with **rx_framesperfile** being the maximum number of frames per file. The default varies for each detector type.
+
+
+Some file name examples:
 
     .. code-block:: bash
 
@@ -108,17 +252,21 @@ File format
         # second acquisition, first file
         path-to-file/run_d0_f0_1.raw
 
-* Each acquisition will create a master file that can be enabled/disabled using **fmaster**. This should have parameters relevant to the acquisition.
 
-* SLS Receiver Header consist of SLS Detector Header + 64 bytes of bitmask, altogether 112 bytes. The packetNumber in the sls detector header part, will be updated to number of packets caught by receiver for that frame. Furthermore, the bit mask will specify which packets have been received.
+Each acquisition will create a master file that can be enabled/disabled using **fmaster**. This should have parameters relevant to the acquisition.
+
+
+SLS Receiver Header consist of SLS Detector Header + 64 bytes of bitmask, altogether 112 bytes. The packetNumber in the sls detector header part, will be updated to number of packets caught by receiver for that frame. Furthermore, the bit mask will specify which packets have been received.
 
 **Binary file format**
 
-* This is the default file format. 
+This is the default file format. 
 
-* Each data file will consist of frames, each consisting of slsReceiver Header followed by data for 1 frame.
 
-* Master file is of ASCII format and will also include the format of the slsReceiver Header.
+Each data file will consist of frames, each consisting of slsReceiver Header followed by data for 1 frame.
+
+
+Master file is of ASCII format and will also include the format of the slsReceiver Header.
 
 
 **HDF5 file formats**
@@ -160,7 +308,7 @@ When handling callbacks, the control should be returned as soon as possible, to
 **Example**
     * `main cpp file <https://github.com/slsdetectorgroup/api-examples/blob/master/e4-receiver_callbacks.cpp>`_ 
     * `cmake file <https://github.com/slsdetectorgroup/api-examples/blob/master/CMakeLists.txt>`_. 
-    * how to install the slsDetectorPackage is provided :ref:`here <build from source using cmake>`.
+    * how to install the slsDetectorPackage with cmake is provided :ref:`here <build from source using cmake>`.
     * compile the example **e4-rxr** by:
 
         .. code-block:: bash

docs/src/troubleshooting.rst

@@ -8,21 +8,24 @@ open an issue at our  `github repo issues
 Common
 ------
 
-Missing Packets
-^^^^^^^^^^^^^^^
-Possible causes could be the following:
 
-#. Receiver PC is not tuned for socket buffer size and input packet queue.
-    * Refer to :ref:`Increase rmem_default, rmem_max and max_backlog<Receiver PC Tuning>`
+1. Total Failure of Packet Delivery
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-#. Wiring
-    * Faulty wiring or connecting cable to incorrect interface.
+#. Data cable plugged into the wrong interface on board (Jungfrau)
+    * Please ensure that the data cable is plugged into the rightmost interface (default for single interface). The inner one is disabled for PCB v1.0 and must be selected via command for PCB v2.0.
 
 #. Link up and speed
-    * Check to see if there is a blue LED on board to signal that the link is up. Check ethtool and find if Link Deteced:Yes and Speed is acceptable (>10k).
+    * Check ethtool and find if Link Deteced:Yes and Speed is acceptable (>10k).
+    * Check to see if the 10G link is up (blue or red LED on board, close to SFP+). If not:
+
+       * Check transeiver and fibers are compatible (all MMF 850nm or all SMF 1030nm)
+       * Check fiber
+       * Check fiber polarity (if short range, unplug the link anywhere, and look at the light/dark pattern: dark has to mate with light)
 
 #. Detector is not acquiring (Not Eiger)
     * Take an acquisition with many images and using the following steps instead of acquire:
+
         .. code-block:: bash
 
             sls_detector_put status start
@@ -30,37 +33,63 @@ Possible causes could be the following:
             # which means the detector is acquiring.
             sls_detector_get framesl 
 
-    .. note ::
-    
-        If you are using multiple modules, the previous command can return -1 because each module will return different values. Then, check for a single module instead: sls_detector_get 0:framesl
+            # If you are using multiple modules, the previous command can return -1 because each module will return different values. Then, check for a single module instead: sls_detector_get 0:framesl
 
-#. Data cable plugged into the wrong interface on board (Jungfrau)
-    * Please ensure that the data cable is plugged into the rightmost interface. The middle one is disabled for PCB v1.0 and must be selected via command for PCB v2.0.
 
-#. Detector is not sending data
-    * Check the board to see if the green LED is blinking next to the data cable, which means that the detector is sending data.
+#. Detector is not sending data (Except Eiger)
+    * Check the board to see if the green LED close to SFP is blinking (detector is sending data). If not, detector is not operated properly (period too short/long, no trigger in trigger mode) or misconfigured and needs reboot.
 
-#. Firewall or security feature
-    * A firewall or some security feature could be blocking the reception of data.
-
-#. Ethernet interface not configured properly
-    * Ensure that the interfaces used are configured properly with the right mask and ip. Eg. use ifconfig and route commands to verify.
+#. Power supply
+    * Check if power supply has enough current. 
+    * For Jungfrau, refer to :ref:`Jungfrau Power Supply Troubleshooting<Jungfrau Troubleshooting Power Supply>`.
 
 #. Ethernet interface not configured for Jumbo frames (10Gb)
-    * Ensure that the interfaces used in receiver pc have MTU 9000 (jumbo frames) enabled.
+    * Ensure that the interfaces (on NIC and the switch) used in receiver pc have MTU 9000 (jumbo frames) enabled.
 
-#. Detector IP (Not Eiger)
-    * Ensure it is valid and does not end if 0 or 255. Also ensure that the detector ip is in the same subnet as rx_udpip and the masking in the interface configuration ensures this rule.
 
-#. Tcpdump or wireshark
+#. Check if 'rx_frames' counter in 'ifconfig' do not increment for interface.
+    * If no, check switch configuration if present. Port counters of switch can also help to identify problem.
+    * If yes, but receiver software does not see it:
+
+        * Check no firewall (eg. firewalld) is present or add rules
+        * Check that selinux is disabled ( or add rules)
+        
+#. Source UDP IP in config file (Not Eiger)
+    * Ensure it is valid and does not end if 0 or 255. Also ensure that the source ip 'udp_srcip' is in the same subnet as destination ip 'udp_dstip' and the masking in the interface configuration ensures this rule.
+    * Eg. If interface IP is 102.10.10.110 and mask is 255.255.255.0, udp_srcip has to be 102.10.10.xxx (same subnet)
+    * Use ifconfig and route commands to verify etheret interface configuration
+
+
+#. Netstat and netcat
+    * Try with netstat to see if its really listening to the right interface. Or netcat to see if you get packets.
+
+#. Wireshark or Tcpdump
     * Use one of these to confirm that you receive packets (with the right filtering ie. source and destination ports, ip).
 
-#. Check SFP modules
-    * Check if the SFP modules on both sides of the fiber are of same type.
+
+
+2. Partial or Random Packet Loss (Performance)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. note ::
+    
+    The following suggestions are for convenience. Please do not follow blindly, research each parameter and adapt it to your system.
+
+#. Receiver PC is not tuned for socket buffer size and input packet queue or other parameters.
+    * Refer to :ref:`Receiver PC Tuning<Receiver PC Tuning>`
+
+#. Wiring
+    * Faulty wiring or connecting cable to incorrect interface.
+
 
 #. Pinging the subnet (receiving only a few number of packets each time)
     * If a switch is used between a receiver pc and detector instead of plugging the cables directly, one might have to ping any ip in the subnet of the Ethernet interface constantly so that it does not forget the ip during operation.
     * Eg. if rx_udpip is 10.2.3.100, then ping constantly 10.2.3.xxx, where xxx is any ip other than 100.
+    * Using slsReceiver, you can use a command that does this for you:
+        .. code-block:: bash
+        
+            # arping the interface in a separate thread every minute
+            sls_detector_put rx_arping 1
 
 
 
@@ -68,6 +97,12 @@ Possible causes could be the following:
 
 Receiver PC Tuning Options
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. note ::
+
+    | xth1 is example interface name in the following examples. 
+    | These settings are lost at pc reboot.
+
 #. Increase maximum receive socket buffer size and socket input packet queue. 
     * Temporarily (until shut down)
         .. code-block:: bash
@@ -111,6 +146,7 @@ Receiver PC Tuning Options
         # check how many GB memory you can allocate, to avoid swapping otherwise    
 
 
+
 #. Modify ethtool settings. 
     * rx ring parameters
         .. code-block:: bash
@@ -127,6 +163,9 @@ Receiver PC Tuning Options
             # check 
             ethtool -c xth1
 
+            # enable adaptive xoalescence parameters
+            ethtool -C xth1 adaptive-rx on
+
             # set to max value in your pc settings
             ethtool -C xth1 rx-usecs 100 
 
@@ -138,11 +177,43 @@ Receiver PC Tuning Options
 
             # set to max value in your pc settings
             ethtool -A xth1 rx on
- 
-    .. note ::
+    
+    * generic receiver offload (might not always work)
+        .. code-block:: bash
 
-        | xth1 is example interface name. 
-        | These settings are lost at pc reboot.
+            # check
+            ethtool -k xth1
+
+            # enable generic receiver offload
+            ethtool -K xth1 gro
+        
+
+#. Disable power saving in CPU frequency scaling and set system to performance 
+    * Check current policy (default might be powersave or schedutil)
+        .. code-block:: bash
+            
+            # check current active governor and range of cpu freq policy
+            cpupower frequency-info --policy
+            # list all available governors for this kernel
+            cpupower frequency-info --governors  
+
+    * Temporarily (until shut down)
+        .. code-block:: bash
+            
+            # set to performance
+            sudo cpupower frequency-set -g performance
+
+            # or
+            cpufreq-info
+            for i in ‘seq 0 7‘; do cpufreq-set -c $i -g performance; done
+            
+    * Permanently
+        .. code-block:: bash
+            
+            # edit /etc/sysconfig/cpupower to preference
+
+            # enable or disable permanently
+            sudo systemctl enable cpupower
 
 #. Give user speicific user scheduling privileges.
     .. code-block:: bash
@@ -155,18 +226,29 @@ Receiver PC Tuning Options
 
         This is also set if slsReceiver is run as root user.
         
+#. Some more advanced options: 
+    .. warning ::
         
-#. Disable power saving in CPU frequency 
-    .. code-block:: bash
+        Please do not try if you do not understand
 
-        # or similar command depending on your distribution
-        cpupower frequency-info
-        cpupower frequency-set -g performance
+    #. reduce the number of queue per NIC to the number of expected streams: ethtool -L xth0 combined 2 
+    #. assign each queue to its stream:  ethtool -U xth0 flow-type tcp4 dst-port 50004 action 1
+    #. assign to each queue (IRQ) one CPU on the right socket:  echo "3"> /proc/irq/47/smp_affinity_list    #change the numbers looking at /proc/interrupts
+    #. disable irqbalance service 
+    #. Be sure that the switch knows the receiver mac address. Most switches reset the mac lists every few minutes, and since the receiver only receives, there is not a periodic refresh of the mac list. In this case, one can set a fixed mac list in the switch, or setup some kind of script arping or pinging out from that interface (will be available in 7.0.0). 
+    #. assign the receiver numa node (also with -m) to the socket where the NIC is attached. To know it, cat /sys/class/net/ethxxx/device/numa_node
+    #. ensure file system performance can handle sustained high data rate:
+        
+        * One can use dd: 
 
-        # or
-        cpufreq-info
-        for i in ‘seq 0 7‘; do cpufreq-set -c $i -g performance; done
+            .. code-block:: bash
+	            
+                dd if=/dev/zero of=/testpath/testfile bs=1M count=100000
+        * Or better fio (which needs to be installed) 
 
+            .. code-block:: bash
+	        
+                fio --name=global –directory=/testpath/ --rw=write --ioengine=libaio --direct=0 --size=200G -- 	numjobs=2 --iodepth=1 --bs=1M –name=job
 
 slsReceiver Tuning
 ^^^^^^^^^^^^^^^^^^
@@ -174,18 +256,25 @@ slsReceiver Tuning
 #. Starting receiver as root to have scheduling privileges.
 
 #. For 10g, enable flow control
+
     .. code-block:: bash
 
         sls_detector_put flowcontrol10g 1
 
-#. Increase slsReceiver fifo depth between listening and processing threads.
+#. Increase slsReceiver ring buffer depth 
+    This can be tuned depending on the number of receivers (modules) and memory available.
+
     .. code-block:: bash
 
-        sls_detector_get rx_fifodepth
-        # sets number of frames in fifo to 5000
-        sls_detector_put rx_fifodepth 5000
+        # sugggested not to use more than half memory of CPU socket in case of NUMA systems) for this
+
+        sls_detector_get rx_fifodepth
+        # sets number of frames in fifo to 1024 ~1GB per receiver. Default is 2500
+        sls_detector_put rx_fifodepth 1024
+
+#. Increase number of frames per file 
+    This can reduce time taken to open and close files.
 
-#. Increase number of frames per file to reduce time taken to open and close files.
     .. code-block:: bash
 
         sls_detector_get rx_framesperfile
@@ -193,10 +282,31 @@ slsReceiver Tuning
         # writes all frames into a single file
         sls_detector_put rx_framesperfile 0
 
+#. Disable file write
+    This can ensure it is not the file system performance hampering high date rate.
+
+    .. code-block:: bash
+
+        sls_detector_put fwrite 0
+
 
 Shared memory error
 ^^^^^^^^^^^^^^^^^^^
-| For errors due to access or size, delete shared memory files nd try again.
+For errors due to access or size, use any of the following suggestions.
+    #. Delete shared memory files and try again
+    #. Use environment variable to use a different shared memory ending in jfxx
+        
+        .. code-block:: bash
+
+            # shared memory ending in jfxx
+            export SLSDETNAME=jfxx
+
+    #. USe a different multi shared memory ID
+        .. code-block:: bash
+    
+            sls_detector_put 2-config xxxx.config
+            # or
+            sls_detector_put 2-hostname bchipxxx
 
 To list all shared memory files of sls detector package.
     .. code-block:: bash
@@ -247,6 +357,19 @@ Possible causes could be the following:
     * For Jungfrau, refer to :ref:`Jungfrau Power Supply Troubleshooting<Jungfrau Troubleshooting Power Supply>`.
 
 
+Cannot ping module (Nios)
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you executed "reboot" command on the board, you cannot ping it anymore unless you power cycle. To reboot the controller, please use the software command ("rebootcontroller"), which talks to the microcontroller.
+
+Gotthard2
+---------
+
+Cannot get data without a module attached
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You cannot get data without a module attached as a specific pin is floating. Attach module to get data.
+
 
 Gotthard
 ----------
@@ -294,6 +417,7 @@ Cannot get multi module data
 
 #. Check :ref:`Common Multi Module Troubleshooting<common troubleshooting multi module data>`
 #. Power Supply
+    * Jungfrau needs a ~4A per module for a short time at startup. If not, it reboots misconfigured.
     * Comment out this line in the config file: powerchip 1
     * Powering on the chip increases the power consumption by a considerable amount. If commenting out this line aids in getting data (strange data due to powered off chip), then it could be the power supply current limit. Fix it (possibly to 8A current limit) and uncomment the powerchip line back in config file.
 

docs/src/udpconfig.rst

@@ -0,0 +1,37 @@
+.. _detector udp header:
+
+
+Config file
+============
+
+Commands to configure the UDP in the config file:
+
+.. note :: 
+
+    These command are recommended to be placed before "rx_hostname" if it is used.
+
+Source Port
+-----------
+    Hardcoded in detector server, starting at 32410.
+
+udp_srcip - Source IP
+---------------------
+    IP address of detector UDP interface to send packets from. Do not use for Eiger 1Gb interface (uses its hardware IP). For others, must be in the same subnet as **udp_dstip**.
+
+udp_srcmac - Source MAC
+-----------------------
+    MAC address of detector UDP interface to send packets from. Do not use for Eiger (uses hardware mac). For others, it is not necessary, but can help for switch and debugging to put unique values for each module.
+ 
+
+udp_dstport - Desintation Port
+-------------------------------
+    Port in receiver pc to listen to packets from the detector.
+
+udp_dstip - Destination IP
+--------------------------
+    IP address of interface in receiver pc to listen to packets from detector. If **auto** is used (only when using slsReceiver/ slsMultiReceiver), the IP of **rx_hostname** is picked up.
+
+udp_dstmac - Destination MAC
+----------------------------
+    MAC address of interface in receiver pc to list to packets from detector. Only required when using custom receiver, else slsReceiver/slsMultiReceiver picks it up from **udp_dstip**.
+

docs/src/udpdetspec.rst

@@ -0,0 +1,152 @@
+.. _detector specific fields:
+
+Detector Specific Fields
+========================
+
+Please check out :ref:`the current detector header <detector udp header>` to see 
+where the detector specific fields are placed.
+
+
+
+Eiger
+------
+
+.. table:: Detector Specific Field
+   
+   +----------+------------------------------+
+   | expLength| Sub Frame Number             |
+   +----------+------------------------------+
+   | detSpec1 | 0x0                          |
+   +----------+------------------------------+
+   | detSpec2 | 0x0                          |
+   +----------+------------------------------+
+   | detSpec3 | e14a                         |
+   +----------+------------------------------+
+   | detSpec4 | Round Robin Interface Number |
+   +----------+------------------------------+
+
+
+Jungfrau
+---------
+
+.. table:: Detector Specific Field
+
+   +----------+------------------------------+
+   | detSpec1 | Bunch Id [#]_                |
+   +----------+------------------------------+
+   | detSpec2 | 0                            |
+   +----------+------------------------------+
+   | detSpec3 | DAQ info                     |
+   +----------+------------------------------+
+   | detSpec4 | 0                            |
+   +----------+------------------------------+
+
+
+.. table:: DAQ Info Field
+
+   +----------+--------------------+----------------------------------------------+
+   |   Bits   |       Name         |   Description                                |
+   +----------+--------------------+-----+----------------------------------------+
+   | 0        | High gain          |  1  | High Gain enabled                      |
+   |          |                    +-----+----------------------------------------+
+   |          |                    |  0  | High Gain disabled                     |
+   +----------+--------------------+-----+----------------------------------------+
+   | 1        | Fix gain stage 1   |  1  | Gain stage 1 fixed. The switch that    |
+   |          |                    |     | selects the gains stage 1 is active all|
+   |          |                    |     | the time.                              |
+   |          |                    +-----+----------------------------------------+
+   |          |                    |  0  | Gain stage 1 unset. The switch that    |
+   |          |                    |     | selects the gains stage 1 is inactive  |
+   |          |                    |     | all the time.                          |
+   +----------+--------------------+-----+----------------------------------------+
+   | 2        | Fix gain stage 2   |  1  | Gain stage 2 fixed. The switch that    |
+   |          |                    |     | selects the gains stage 2 is active all|
+   |          |                    |     | the time.                              |
+   |          |                    +-----+----------------------------------------+
+   |          |                    |  0  | Gain stage 2 unset. The switch that    |
+   |          |                    |     | selects the gains stage 2 is inactive  |
+   |          |                    |     | all the time.                          |
+   +----------+--------------------+-----+----------------------------------------+
+   | 4        | Comparator reset   |  1  | On-chip comparator in reset state.     | 
+   |          |                    |     | Dynamic-gain switching is therefore    |
+   |          |                    |     | disabled.                              |
+   |          |                    +-----+----------------------------------------+
+   |          |                    |  0  | On-chip comparator active.             |
+   +----------+--------------------+-----+-----+-----+----------------------------+
+   | 7-5      | Jungfrau chip      |Bit 7|Bit 6|Bit 5| Description                |
+   |          | version            +-----+-----+-----+----------------------------+
+   |          |                    | 0   |   0 |  0  | v1.0                       |
+   |          |                    +-----+-----+-----+----------------------------+
+   |          |                    | 0   |   0 |  1  | v1.1                       |
+   |          |                    +-----+-----+-----+----------------------------+
+   |          |                    | 0   |   1 |  X  | Reserved                   |
+   |          |                    +-----+-----+-----+----------------------------+
+   |          |                    | 1   |   X |  X  | Reserved                   |
+   +----------+--------------------+-----+-----+-----+----------------------------+
+   | 11-8     | Storage cell select|Storage cell used for this exposure. This     |
+   |          |                    |field defines the storage cell that was used  |
+   |          |                    |to acquire the data of this frame             |
+   +----------+--------------------+-----+----------------------------------------+
+   | 12       | Force switching    |  1  | Forced switching to gain stage 1 at the|
+   |          | to gain stage 1    |     | start of the exposure period.          |
+   |          |                    +-----+----------------------------------------+
+   |          |                    |  0  | Disabled forced gain switching to gain |
+   |          |                    |     | stage 1. Dynamic gain switching        |
+   |          |                    |     | conditions apply.                      |
+   +----------+--------------------+-----+----------------------------------------+
+   | 13       | Force switching    |  1  | Forced switching to gain stage 2 at the|
+   |          | to gain stage 2    |     | start of the exposure period.          |
+   |          |                    +-----+----------------------------------------+
+   |          |                    | 0   | Disabled forced gain switching to gain |
+   |          |                    |     | stage 2. Dynamic gain switching        |
+   |          |                    |     | conditions apply.                      |
+   +----------+--------------------+-----+-----+-----+----------------------------+
+   | 23-16    |  10-Gigabit event  |The 8-bit event code contains value of the    |
+   |          |  code              |event received over the 10 GbE interface by   |
+   |          |                    |JUNGFRAU detector at the moment of the frame  |
+   |          |                    |acquisition.                                  |
+   +----------+--------------------+-----+----------------------------------------+
+   | 31       | External input flag|  1  | External input flag detected in the    |
+   |          |                    |     | last exposure.                         |
+   |          |                    +-----+----------------------------------------+
+   |          |                    |  0  | External input flag not detected in the|
+   |          |                    |     | last exposure.                         |
+   +----------+--------------------+-----+----------------------------------------+
+
+
+
+Gotthard2
+----------
+
+.. table:: Detector Specific Field
+
+   +----------+------------------------------+
+   | detSpec1 | Train Id [#]_                |
+   +----------+------------------------------+
+   | detSpec2 | Bunch Id [#]_                |
+   +----------+------------------------------+
+   | detSpec3 | 0                            |
+   +----------+------------------------------+
+   | detSpec4 | 0                            |
+   +----------+------------------------------+
+
+
+Mythen3
+----------
+
+.. table:: Detector Specific Field
+
+   +----------+------------------------------+
+   | detSpec1 | 0                            |
+   +----------+------------------------------+
+   | detSpec2 | 0                            |
+   +----------+------------------------------+
+   | detSpec3 | 0                            |
+   +----------+------------------------------+
+   | detSpec4 | 0                            |
+   +----------+------------------------------+
+
+
+.. [#] **Bunch Id**: bunch identification number received by the detector at the moment of frame acquisition.
+.. [#] **Train Id**: train identification number received by the detector at the moment of frame acquisition.
+.. [#] **Bunch Id**: bunch identification number to identify every single exposure during a burst acquisition.

docs/src/udpheader.rst

@@ -0,0 +1,125 @@
+.. _detector udp header:
+
+Format
+=======
+
+The UDP data format for the packets consist of a common header for all detectors, followed by the data for that one packet.
+
+
+Current Version
+---------------------------
+
+**v2.0 (slsDetectorPackage v7.0.0+)**
+
+.. table:: <---------------------------------------------------- 8 bytes ---------------------------------------------------->
+    :align: center
+    :widths: 30,30,30,15,15
+
+    +---------------------------------------------------------------+
+    |                          frameNumber                          |
+    +-------------------------------+-------------------------------+
+    |            expLength          |         packetNumber          |
+    +-------------------------------+-------------------------------+
+    |                         **detSpec1**                          |
+    +---------------------------------------------------------------+
+    |                           timestamp                           |
+    +---------------+---------------+---------------+---------------+
+    |     modId     |      row      |     column    |  **detSpec2** |
+    +---------------+---------------+---------------+-------+-------+
+    |          **detSpec3**         |  **detSpec4** |detType|version|
+    +-------------------------------+---------------+-------+-------+
+
+
+.. note :: 
+
+    Since there is no difference in the format of the UDP header from the detector
+    from the previous version (v2.0), the version number stays the same.
+    
+    Only the struture member names have changed in sls_detector_defs.h
+
+
+
+Description
+------------
+
+* **Detector specific field** descriptions are found :ref:`here<detector specific fields>`.
+
+* **frameNumber**: framenumber to which the current packet belongs to.
+
+* **expLength**: measured exposure time of the frame in tenths of microsecond. It is instead the sub frame number for Eiger.
+
+* **packetNumber**: packet number of the frame to which the current data belongs to.
+
+* **timestamp**: time measured at the start of frame exposure since the start of the current measurement. It is expressed in tenths of microsecond.
+
+* **modId**: module ID picked up from det_id_[detector type].txt on the detector cpu.
+
+* **row**: row position of the module in the detector system. It is calculated by the order of the module in hostname command, as well as the detsize command.
+
+* **column**: column position of the module in the detector system.  It is calculated by the order of the module in hostname command, as well as the detsize command.
+
+* **detType**: detector type from enum of detectorType in the package.
+
+* **version**: current version of the detector header (0x2).
+
+
+Detector Enum
+--------------
+
+    ================    ========
+    Detector Type        Value
+    ================    ========
+    GENERIC             0
+    EIGER               1
+    GOTTHARD            2    
+    JUNGFRAU            3    
+    CHIPTESTBOARD       4        
+    MOENCH              5
+    MYTHEN3             6
+    GOTTHARD2           7    
+    ================    ========
+
+
+
+Previous Versions
+-----------------
+**v2.0 (Package v4.0.0 -  6.x.x)**
+
+.. table:: <---------------------------------------------------- 8 bytes ---------------------------------------------------->
+    :align: center
+    :widths: 30,30,30,15,15
+
+    +---------------------------------------------------------------+
+    |                          frameNumber                          |
+    +-------------------------------+-------------------------------+
+    |            expLength          |         packetNumber          |
+    +-------------------------------+-------------------------------+
+    |                            bunchid                            |
+    +---------------------------------------------------------------+
+    |                           timestamp                           |
+    +---------------+---------------+---------------+---------------+
+    |     modId     |    **row**    |   **column**  |  **reserved** |
+    +---------------+---------------+---------------+-------+-------+
+    |             debug             |  roundRNumber |detType|version|
+    +-------------------------------+---------------+-------+-------+
+
+**v1.0 (Package v3.0.0 -  3.1.5)**
+
+.. table:: <---------------------------------------------------- 8 bytes ---------------------------------------------------->
+    :align: center
+    :widths: 30,30,30,15,15
+
+    +---------------------------------------------------------------+
+    |                          frameNumber                          |
+    +-------------------------------+-------------------------------+
+    |            expLength          |         packetNumber          |
+    +-------------------------------+-------------------------------+
+    |                            bunchid                            |
+    +---------------------------------------------------------------+
+    |                           timestamp                           |
+    +---------------+---------------+---------------+---------------+
+    |     modId     |    xCoord     |     yCoord    |    zCoord     |
+    +---------------+---------------+---------------+-------+-------+
+    |             debug             |  roundRNumber |detType|version|
+    +-------------------------------+---------------+-------+-------+
+

docs/src/virtualserver.rst

@@ -1,6 +1,6 @@
 .. _Virtual Detector Servers:
-Detector Simulators
-===================
+Simulators
+===========
 
 Compilation
 -----------
@@ -13,16 +13,14 @@ Compilation
 * Using cmk.sh script,
      .. code-block:: bash  
         
-        ./cmk.sh -bsj9 #option s is for simulator
+        ./cmk.sh -bsj9 # option -s is for simulator
 
 
 Binaries
 ^^^^^^^^
     .. code-block:: bash  
 
-        eigerDetectorServerMaster_virtual
-        eigerDetectorServerSlaveTop_virtual
-        eigerDetectorServerSlaveBottom_virtual
+        eigerDetectorServer_virtual
         jungfrauDetectorServer_virtual
         gotthardDetectorServer_virtual
         gotthard2DetectorServer_virtual
@@ -67,7 +65,15 @@ Sample Config file
 ^^^^^^^^^^^^^^^^^^
 There are sample config files for each detector in slsDetectorPackage/examples folder.
 
-For a Single Module
+For a Single Module (Basic)
+    .. code-block:: bash  
+
+        hostname localhost
+        rx_hostname localhost
+        udp_dstip auto
+
+
+For a Single Module (With Options)
     .. code-block:: bash  
 
         # connects to control port 1912
@@ -140,7 +146,7 @@ Gui
 Limitations
 -----------
 
-#. Data coming out of virtual server is fake. Value at each pixel/ channel is incremented by 1.
+#. Data coming out of virtual server is fake. 
 
 #. A stop will stop the virtual acquisition only at the start of every new frame.
 

docs/static/extra.css

@@ -1,4 +1,4 @@
 /* override table no-wrap */
 .wy-table-responsive table td, .wy-table-responsive table th {
     white-space: normal;
-}
+}

evalVersionVariables.sh

@@ -1,22 +0,0 @@
-GITREPO1='git remote -v'
-GITREPO2=" | grep \"fetch\" | cut -d' ' -f1"
-BRANCH1='git branch -v'
-BRANCH2=" | grep '*' | cut -d' ' -f2"
-REPUID1='git log --pretty=format:"%H" -1'
-AUTH1_1='git log --pretty=format:"%cn" -1'
-AUTH1_2=" | cut -d' ' -f1"
-AUTH2_1='git log --pretty=format:"%cn" -1'
-AUTH2_2=" | cut -d' ' -f2"
-FOLDERREV1='git log --oneline . '   #used for all the individual server folders
-FOLDERREV2=" | wc -l"  #used for all the individual server folders
-REV1='git log --oneline  '
-REV2=" | wc -l"
-
-GITREPO=`eval $GITREPO1  $GITREPO2`
-BRANCH=`eval $BRANCH1  $BRANCH2`
-REPUID=`eval $REPUID1`
-AUTH1=`eval $AUTH1_1  $AUTH1_2`
-AUTH2=`eval $AUTH2_1  $AUTH2_2`
-REV=`eval $REV1  $REV2`
-FOLDERREV=`eval $FOLDERREV1  $FOLDERREV2`
-

examples/badchannel.txt

@@ -0,0 +1,6 @@
+0
+10, 30
+40:45 50:52
+1279
+# all bad channels are applied for all counters in deector
+

examples/jctb_moench03_T1.config

@@ -409,18 +409,18 @@ patword 018d 0008599f0008503a
 patioctrl 8f0effff6dbffdbf
 patclkctrl 0000000000000000
 patlimits 0000 018c
-patloop0 013a 016b
-patnloop0 199
-patloop1 0400 0400
-patnloop1 0
-patloop2 0400 0400
-patnloop2 0
-patwait0 00aa
-patwaittime0 10000
-patwait1 0400
-patwaittime1 0
-patwait2 0400
-patwaittime2 0
+patloop 0 013a 016b
+patnloop 0 199
+patloop 1 0400 0400
+patnloop 1 0
+patloop 2 0400 0400
+patnloop 2 0
+patwait 0 00aa
+patwaittime 0 10000
+patwait 1 0400
+patwaittime 1 0
+patwait 2 0400
+patwaittime 2 0
 
 #############################################
 ### edit with hostname or 1Gbs IP address of your server

examples/jungfrau_two.config

@@ -5,7 +5,7 @@ detsize 1024 1024
 hostname bchip048+bchip052+
 
 # 1Gb receiver pc hostname  (default tcpport: 1954)
-rx_hostname pcmoench01+pcmoench01:1955
+rx_hostname pcmoench01:1954+pcmoench01:1955+
 
 
 

examples/moench03_T1.config

@@ -3,7 +3,7 @@
 ### edit with hostname or IP address of your detector 
 ############################################
 #hostname bchip181+
-hostname bchip181+
+hostname bchip135 
 
 #############################################
 ### edit with hostname or 1Gbs IP address of your server
@@ -28,7 +28,7 @@ rx_zmqport 50003
 #############################################
 ### edit with 1 Gbs IP of PC where you will run the GUI
 ############################################
-zmqip 129.129.202.98
+zmqip 129.129.202.57
 zmqport 50001
 
 

examples/virtual_ctb_moench.config

@@ -427,18 +427,18 @@ patword 0x018c 0x0008599f0008503a
 patword 0x018d 0x0008599f0008503a
 patioctrl 0x8f0effff6dbffdbf
 patlimits 0x0000 0x018c
-patloop0 0x013a 0x016b
-patnloop0 0x199
-patloop1 0x0400 0x0400
-patnloop1 0
-patloop2 0x0400 0x0400
-patnloop2 0
-patwait0 0x00aa
-patwaittime0 10000
-patwait1 0x0400
-patwaittime1 0
-patwait2 0x0400
-patwaittime2 0
+patloop 0 0x013a 0x016b
+patnloop 0 0x199
+patloop 1 0x0400 0x0400
+patnloop 1 0
+patloop 2 0x0400 0x0400
+patnloop 2 0
+patwait 0 0x00aa
+patwaittime 0 10000
+patwait 1 0x0400
+patwaittime 1 0
+patwait 2 0x0400
+patwaittime 2 0
 
 # dacs
 dac 6 800

genVersionHeader.sh

@@ -1,25 +0,0 @@
-#####! /bin/awk -f
-
-
-if [ $# -lt 3 ] 
-then
-    echo "wrong usage"
-    exit -1
-fi
-
-
-fin=$1
-ftmp=$2
-fout=$3
-
-
-#dat=echo "date '+%Y%m%d'"
-
-echo "Updating $fout"
-#echo "in: $fin tmp: $ftmp out: $fout"
-
-#awk 'NR==FNR {if ($3=="Date:") {l[FNR]=$4; gsub("-","",l[FNR]);} else { if (match($0,"Rev")) {l[FNR]=$(NF);} else {l[FNR]="\""$(NF)"\"";};};next} {$0=$1" "$2" "l[FNR]}1' $fin $ftmp > $fout
-
-awk 'BEGIN {l[0]=0; "date +%Y%m%d" | getline l[1]; l[2]="\"/\""; l[3]="\"nobody\""; l[3]="\"nobody\""; l[4]="\"0000-0000-0000\"";} \
-NR==FNR {if (match($0,"Rev")) {l[0]="0x"$(NF);} else if (match($0,"Date")) {l[1]="0x"$4; gsub("-","",l[1]);} else if (match($0,"URL")) {l[2]="\""$(NF)"\"";} else if (match($0,"Author")) {l[3]="\""$(NF)"\"";} else if (match($0,"UUID")) {l[4]="\""$(NF)"\"";} else if (match($0,"Branch")) {l[5]="\""$(NF)"\"";};next;} 
-{if (match($2,"REV")) {$0=$1" "$2" "l[0];} else if (match($2,"DATE")) {$0=$1" "$2" "l[1];} else if (match($2,"URL")) {$0=$1" "$2" "l[2];} else if (match($2,"AUTH")) {$0=$1" "$2" "l[3];} else if (match($2,"UUID")) {$0=$1" "$2" "l[4];} else if (match($2,"BRANCH")) {$0=$1" "$2" "l[5];}}1' $fin $ftmp > $fout

integrationTests/CMakeLists.txt

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: LGPL-3.0-or-other
+# Copyright (C) 2021 Contributors to the SLS Detector Package
 # MESSAGE( STATUS "CMAKE_CURRENT_SOURCE_DIR: " ${CMAKE_CURRENT_SOURCE_DIR} )
 # MESSAGE( STATUS "PROJECT_SOURCE_DIR: " ${PROJECT_SOURCE_DIR} )
 

integrationTests/test-eigerIntegration.cpp

@@ -1,9 +1,13 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #include "DetectorImpl.h"
 #include "catch.hpp"
 #include "sls/string_utils.h"
 #include "tests/globals.h"
 #include <iostream>
 
+namespace sls {
+
 class MultiDetectorFixture {
   protected:
     DetectorImpl d;
@@ -134,7 +138,7 @@ TEST_CASE_METHOD(MultiDetectorFixture, "Get ID", "[.eigerintegration][cli]") {
     std::string hn = test::hostname;
     hn.erase(std::remove(begin(hn), end(hn), 'b'), end(hn));
     hn.erase(std::remove(begin(hn), end(hn), 'e'), end(hn));
-    auto hostnames = sls::split(hn, '+');
+    auto hostnames = split(hn, '+');
     CHECK(hostnames.size() == d.getNumberOfDetectors());
     for (int i = 0; i != d.getNumberOfDetectors(); ++i) {
         CHECK(d.getId(defs::DETECTOR_SERIAL_NUMBER, 0) ==
@@ -196,3 +200,5 @@ TEST_CASE_METHOD(MultiDetectorFixture, "rate correction",
     d.setRateCorrection(200);
     CHECK(d.getRateCorrection() == 200);
 }
+
+} // namespace sls

integrationTests/test-integrationDectector.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 
 #include "catch.hpp"
 
@@ -22,6 +24,8 @@
 // extern std::string detector_type;
 // extern dt type;
 
+namespace sls {
+
 TEST_CASE("Single detector no receiver", "[.integration][.single]") {
     auto t = Module::getTypeFromDetector(test::hostname);
     CHECK(t == test::type);
@@ -46,8 +50,8 @@ TEST_CASE("Set control port then create a new object with this control port",
     Is this the best way to initialize the detectors
     Using braces to make the object go out of scope
     */
-    int old_cport = DEFAULT_PORTNO;
-    int old_sport = DEFAULT_PORTNO + 1;
+    int old_cport = DEFAULT_TCP_CNTRL_PORTNO;
+    int old_sport = DEFAULT_TCP_STOP_PORTNO;
     int new_cport = 1993;
     int new_sport = 2000;
     {
@@ -75,7 +79,7 @@ TEST_CASE("Set control port then create a new object with this control port",
 
     Module d(test::type);
     d.setHostname(test::hostname);
-    CHECK(d.getStopPort() == DEFAULT_PORTNO + 1);
+    CHECK(d.getStopPort() == DEFAULT_TCP_STOP_PORTNO);
     d.freeSharedMemory();
 }
 
@@ -281,14 +285,14 @@ TEST_CASE(
     CHECK(m.getRateCorrection() == ratecorr);
 
     // ratecorr fail with dr 4 or 8
-    CHECK_THROWS_AS(m.setDynamicRange(8), sls::RuntimeError);
+    CHECK_THROWS_AS(m.setDynamicRange(8), RuntimeError);
     CHECK(m.getRateCorrection() == 0);
     m.setDynamicRange(16);
     m.setDynamicRange(16);
     m.setRateCorrection(ratecorr);
     m.setDynamicRange(16);
     m.setRateCorrection(ratecorr);
-    CHECK_THROWS_AS(m.setDynamicRange(4), sls::RuntimeError);
+    CHECK_THROWS_AS(m.setDynamicRange(4), RuntimeError);
     CHECK(m.getRateCorrection() == 0);
 }
 
@@ -327,11 +331,11 @@ TEST_CASE("Chiptestboard Loading Patterns", "[.ctbintegration]") {
     m.setPatternWord(addr, word);
     CHECK(m.setPatternWord(addr, -1) == word);
     addr = MAX_ADDR;
-    CHECK_THROWS_AS(m.setPatternWord(addr, word), sls::RuntimeError);
+    CHECK_THROWS_AS(m.setPatternWord(addr, word), RuntimeError);
     CHECK_THROWS_WITH(m.setPatternWord(addr, word),
                       Catch::Matchers::Contains("be between 0 and"));
     addr = -1;
-    CHECK_THROWS_AS(m.setPatternWord(addr, word), sls::RuntimeError);
+    CHECK_THROWS_AS(m.setPatternWord(addr, word), RuntimeError);
     CHECK_THROWS_WITH(m.setPatternWord(addr, word),
                       Catch::Matchers::Contains("be between 0 and"));
 
@@ -406,7 +410,7 @@ TEST_CASE("Chiptestboard Dbit offset, list, sampling, advinvert",
     CHECK(m.getReceiverDbitList().size() == 10);
 
     list.push_back(64);
-    CHECK_THROWS_AS(m.setReceiverDbitList(list), sls::RuntimeError);
+    CHECK_THROWS_AS(m.setReceiverDbitList(list), RuntimeError);
     CHECK_THROWS_WITH(m.setReceiverDbitList(list),
                       Catch::Matchers::Contains("be between 0 and 63"));
 
@@ -474,7 +478,7 @@ TEST_CASE("Eiger or Jungfrau nextframenumber",
     CHECK(m.acquire() == slsDetectorDefs::OK);
     CHECK(m.getReceiverCurrentFrameIndex() == val);
 
-    CHECK_THROWS_AS(m.setNextFrameNumber(0), sls::RuntimeError);
+    CHECK_THROWS_AS(m.setNextFrameNumber(0), RuntimeError);
 
     if (m.getDetectorTypeAsString() == "Eiger") {
         val = 281474976710655;
@@ -509,8 +513,10 @@ TEST_CASE("Eiger partialread", "[.eigerintegration][partialread]") {
     m.setDynamicRange(8);
     m.setPartialReadout(256);
     CHECK(m.getPartialReadout() == 256);
-    CHECK_THROWS_AS(m.setPartialReadout(1), sls::RuntimeError);
+    CHECK_THROWS_AS(m.setPartialReadout(1), RuntimeError);
     CHECK(m.getPartialReadout() == 256);
-    CHECK_THROWS_AS(m.setPartialReadout(0), sls::RuntimeError);
+    CHECK_THROWS_AS(m.setPartialReadout(0), RuntimeError);
     m.setPartialReadout(256);
-}
+}
+
+} // namespace sls

integrationTests/test-integrationMulti.cpp

@@ -1,13 +1,17 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 #include "DetectorImpl.h"
 #include "catch.hpp"
 #include "sls/string_utils.h"
 #include "tests/globals.h"
 #include <iostream>
 
+namespace sls {
+
 using namespace Catch::literals;
 
 TEST_CASE("Initialize a multi detector", "[.integration][.multi]") {
-    auto hostnames = sls::split(test::hostname, '+');
+    auto hostnames = split(test::hostname, '+');
 
     DetectorImpl d(0, true, true);
     d.setHostname(test::hostname.c_str());
@@ -100,3 +104,5 @@ TEST_CASE("Set and read timers", "[.integration][.multi]") {
 
     d.freeSharedMemory();
 }
+
+} // namespace sls

integrationTests/test.cpp

@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
 // tests-main.cpp
 #define CATCH_CONFIG_MAIN
 #include "catch.hpp"

libs/pybind/CMakeLists.txt

@@ -0,0 +1,299 @@
+# CMakeLists.txt -- Build system for the pybind11 modules
+#
+# Copyright (c) 2015 Wenzel Jakob <wenzel@inf.ethz.ch>
+#
+# All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+cmake_minimum_required(VERSION 3.4)
+
+# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with
+# some versions of VS that have a patched CMake 3.11. This forces us to emulate
+# the behavior using the following workaround:
+if(${CMAKE_VERSION} VERSION_LESS 3.22)
+  cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
+else()
+  cmake_policy(VERSION 3.22)
+endif()
+
+# Avoid infinite recursion if tests include this as a subdirectory
+if(DEFINED PYBIND11_MASTER_PROJECT)
+  return()
+endif()
+
+# Extract project version from source
+file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/include/pybind11/detail/common.h"
+     pybind11_version_defines REGEX "#define PYBIND11_VERSION_(MAJOR|MINOR|PATCH) ")
+
+foreach(ver ${pybind11_version_defines})
+  if(ver MATCHES [[#define PYBIND11_VERSION_(MAJOR|MINOR|PATCH) +([^ ]+)$]])
+    set(PYBIND11_VERSION_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}")
+  endif()
+endforeach()
+
+if(PYBIND11_VERSION_PATCH MATCHES [[\.([a-zA-Z0-9]+)$]])
+  set(pybind11_VERSION_TYPE "${CMAKE_MATCH_1}")
+endif()
+string(REGEX MATCH "^[0-9]+" PYBIND11_VERSION_PATCH "${PYBIND11_VERSION_PATCH}")
+
+project(
+  pybind11
+  LANGUAGES CXX
+  VERSION "${PYBIND11_VERSION_MAJOR}.${PYBIND11_VERSION_MINOR}.${PYBIND11_VERSION_PATCH}")
+
+# Standard includes
+include(GNUInstallDirs)
+include(CMakePackageConfigHelpers)
+include(CMakeDependentOption)
+
+if(NOT pybind11_FIND_QUIETLY)
+  message(STATUS "pybind11 v${pybind11_VERSION} ${pybind11_VERSION_TYPE}")
+endif()
+
+# Check if pybind11 is being used directly or via add_subdirectory
+if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
+  ### Warn if not an out-of-source builds
+  if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
+    set(lines
+        "You are building in-place. If that is not what you intended to "
+        "do, you can clean the source directory with:\n"
+        "rm -r CMakeCache.txt CMakeFiles/ cmake_uninstall.cmake pybind11Config.cmake "
+        "pybind11ConfigVersion.cmake tests/CMakeFiles/\n")
+    message(AUTHOR_WARNING ${lines})
+  endif()
+
+  set(PYBIND11_MASTER_PROJECT ON)
+
+  if(OSX AND CMAKE_VERSION VERSION_LESS 3.7)
+    # Bug in macOS CMake < 3.7 is unable to download catch
+    message(WARNING "CMAKE 3.7+ needed on macOS to download catch, and newer HIGHLY recommended")
+  elseif(WINDOWS AND CMAKE_VERSION VERSION_LESS 3.8)
+    # Only tested with 3.8+ in CI.
+    message(WARNING "CMAKE 3.8+ tested on Windows, previous versions untested")
+  endif()
+
+  message(STATUS "CMake ${CMAKE_VERSION}")
+
+  if(CMAKE_CXX_STANDARD)
+    set(CMAKE_CXX_EXTENSIONS OFF)
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
+  endif()
+
+  set(pybind11_system "")
+
+  set_property(GLOBAL PROPERTY USE_FOLDERS ON)
+else()
+  set(PYBIND11_MASTER_PROJECT OFF)
+  set(pybind11_system SYSTEM)
+endif()
+
+# Options
+option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT})
+option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT})
+option(PYBIND11_NOPYTHON "Disable search for Python" OFF)
+set(PYBIND11_INTERNALS_VERSION
+    ""
+    CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
+
+cmake_dependent_option(
+  USE_PYTHON_INCLUDE_DIR
+  "Install pybind11 headers in Python include directory instead of default installation prefix"
+  OFF "PYBIND11_INSTALL" OFF)
+
+cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" OFF
+                       "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF)
+
+# NB: when adding a header don't forget to also add it to setup.py
+set(PYBIND11_HEADERS
+    include/pybind11/detail/class.h
+    include/pybind11/detail/common.h
+    include/pybind11/detail/descr.h
+    include/pybind11/detail/init.h
+    include/pybind11/detail/internals.h
+    include/pybind11/detail/type_caster_base.h
+    include/pybind11/detail/typeid.h
+    include/pybind11/attr.h
+    include/pybind11/buffer_info.h
+    include/pybind11/cast.h
+    include/pybind11/chrono.h
+    include/pybind11/common.h
+    include/pybind11/complex.h
+    include/pybind11/options.h
+    include/pybind11/eigen.h
+    include/pybind11/embed.h
+    include/pybind11/eval.h
+    include/pybind11/gil.h
+    include/pybind11/iostream.h
+    include/pybind11/functional.h
+    include/pybind11/numpy.h
+    include/pybind11/operators.h
+    include/pybind11/pybind11.h
+    include/pybind11/pytypes.h
+    include/pybind11/stl.h
+    include/pybind11/stl_bind.h
+    include/pybind11/stl/filesystem.h)
+
+# Compare with grep and warn if mismatched
+if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12)
+  file(
+    GLOB_RECURSE _pybind11_header_check
+    LIST_DIRECTORIES false
+    RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
+    CONFIGURE_DEPENDS "include/pybind11/*.h")
+  set(_pybind11_here_only ${PYBIND11_HEADERS})
+  set(_pybind11_disk_only ${_pybind11_header_check})
+  list(REMOVE_ITEM _pybind11_here_only ${_pybind11_header_check})
+  list(REMOVE_ITEM _pybind11_disk_only ${PYBIND11_HEADERS})
+  if(_pybind11_here_only)
+    message(AUTHOR_WARNING "PYBIND11_HEADERS has extra files:" ${_pybind11_here_only})
+  endif()
+  if(_pybind11_disk_only)
+    message(AUTHOR_WARNING "PYBIND11_HEADERS is missing files:" ${_pybind11_disk_only})
+  endif()
+endif()
+
+# CMake 3.12 added list(TRANSFORM <list> PREPEND
+# But we can't use it yet
+string(REPLACE "include/" "${CMAKE_CURRENT_SOURCE_DIR}/include/" PYBIND11_HEADERS
+               "${PYBIND11_HEADERS}")
+
+# Cache variable so this can be used in parent projects
+set(pybind11_INCLUDE_DIR
+    "${CMAKE_CURRENT_LIST_DIR}/include"
+    CACHE INTERNAL "Directory where pybind11 headers are located")
+
+# Backward compatible variable for add_subdirectory mode
+if(NOT PYBIND11_MASTER_PROJECT)
+  set(PYBIND11_INCLUDE_DIR
+      "${pybind11_INCLUDE_DIR}"
+      CACHE INTERNAL "")
+endif()
+
+# Note: when creating targets, you cannot use if statements at configure time -
+# you need generator expressions, because those will be placed in the target file.
+# You can also place ifs *in* the Config.in, but not here.
+
+# This section builds targets, but does *not* touch Python
+# Non-IMPORT targets cannot be defined twice
+if(NOT TARGET pybind11_headers)
+  # Build the headers-only target (no Python included):
+  # (long name used here to keep this from clashing in subdirectory mode)
+  add_library(pybind11_headers INTERFACE)
+  add_library(pybind11::pybind11_headers ALIAS pybind11_headers) # to match exported target
+  add_library(pybind11::headers ALIAS pybind11_headers) # easier to use/remember
+
+  target_include_directories(
+    pybind11_headers ${pybind11_system} INTERFACE $<BUILD_INTERFACE:${pybind11_INCLUDE_DIR}>
+                                                  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
+
+  target_compile_features(pybind11_headers INTERFACE cxx_inheriting_constructors cxx_user_literals
+                                                     cxx_right_angle_brackets)
+  if(NOT "${PYBIND11_INTERNALS_VERSION}" STREQUAL "")
+    target_compile_definitions(
+      pybind11_headers INTERFACE "PYBIND11_INTERNALS_VERSION=${PYBIND11_INTERNALS_VERSION}")
+  endif()
+else()
+  # It is invalid to install a target twice, too.
+  set(PYBIND11_INSTALL OFF)
+endif()
+
+include("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11Common.cmake")
+
+# Relative directory setting
+if(USE_PYTHON_INCLUDE_DIR AND DEFINED Python_INCLUDE_DIRS)
+  file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${Python_INCLUDE_DIRS})
+elseif(USE_PYTHON_INCLUDE_DIR AND DEFINED PYTHON_INCLUDE_DIR)
+  file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${PYTHON_INCLUDE_DIRS})
+endif()
+
+if(PYBIND11_INSTALL)
+  install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  set(PYBIND11_CMAKECONFIG_INSTALL_DIR
+      "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}"
+      CACHE STRING "install path for pybind11Config.cmake")
+
+  if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}")
+    set(pybind11_INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
+  else()
+    set(pybind11_INCLUDEDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_INCLUDEDIR}")
+  endif()
+
+  configure_package_config_file(
+    tools/${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+    INSTALL_DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
+
+  if(CMAKE_VERSION VERSION_LESS 3.14)
+    # Remove CMAKE_SIZEOF_VOID_P from ConfigVersion.cmake since the library does
+    # not depend on architecture specific settings or libraries.
+    set(_PYBIND11_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P})
+    unset(CMAKE_SIZEOF_VOID_P)
+
+    write_basic_package_version_file(
+      ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
+      VERSION ${PROJECT_VERSION}
+      COMPATIBILITY AnyNewerVersion)
+
+    set(CMAKE_SIZEOF_VOID_P ${_PYBIND11_CMAKE_SIZEOF_VOID_P})
+  else()
+    # CMake 3.14+ natively supports header-only libraries
+    write_basic_package_version_file(
+      ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
+      VERSION ${PROJECT_VERSION}
+      COMPATIBILITY AnyNewerVersion ARCH_INDEPENDENT)
+  endif()
+
+  install(
+    FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake
+          ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
+          tools/FindPythonLibsNew.cmake
+          tools/pybind11Common.cmake
+          tools/pybind11Tools.cmake
+          tools/pybind11NewTools.cmake
+    DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
+
+  if(NOT PYBIND11_EXPORT_NAME)
+    set(PYBIND11_EXPORT_NAME "${PROJECT_NAME}Targets")
+  endif()
+
+  install(TARGETS pybind11_headers EXPORT "${PYBIND11_EXPORT_NAME}")
+
+  install(
+    EXPORT "${PYBIND11_EXPORT_NAME}"
+    NAMESPACE "pybind11::"
+    DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
+
+  # Uninstall target
+  if(PYBIND11_MASTER_PROJECT)
+    configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake_uninstall.cmake.in"
+                   "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY)
+
+    add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P
+                                        ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
+  endif()
+endif()
+
+# BUILD_TESTING takes priority, but only if this is the master project
+if(PYBIND11_MASTER_PROJECT AND DEFINED BUILD_TESTING)
+  if(BUILD_TESTING)
+    if(_pybind11_nopython)
+      message(FATAL_ERROR "Cannot activate tests in NOPYTHON mode")
+    else()
+      add_subdirectory(tests)
+    endif()
+  endif()
+else()
+  if(PYBIND11_TEST)
+    if(_pybind11_nopython)
+      message(FATAL_ERROR "Cannot activate tests in NOPYTHON mode")
+    else()
+      add_subdirectory(tests)
+    endif()
+  endif()
+endif()
+
+# Better symmetry with find_package(pybind11 CONFIG) mode.
+if(NOT PYBIND11_MASTER_PROJECT)
+  set(pybind11_FOUND
+      TRUE
+      CACHE INTERNAL "True if pybind11 and all required components found on the system")
+endif()

libs/pybind/LICENSE

@@ -0,0 +1,29 @@
+Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch>, All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Please also refer to the file .github/CONTRIBUTING.md, which clarifies licensing of
+external contributions to this project including patches, pull requests, etc.

libs/pybind/MANIFEST.in

@@ -0,0 +1,5 @@
+recursive-include pybind11/include/pybind11 *.h
+recursive-include pybind11 *.py
+recursive-include pybind11 py.typed
+include pybind11/share/cmake/pybind11/*.cmake
+include LICENSE README.rst pyproject.toml setup.py setup.cfg

libs/pybind/README.rst

@@ -0,0 +1,180 @@
+.. figure:: https://github.com/pybind/pybind11/raw/master/docs/pybind11-logo.png
+   :alt: pybind11 logo
+
+**pybind11 — Seamless operability between C++11 and Python**
+
+|Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |GitHub Discussions| |CI| |Build status|
+
+|Repology| |PyPI package| |Conda-forge| |Python Versions|
+
+`Setuptools example <https://github.com/pybind/python_example>`_
+• `Scikit-build example <https://github.com/pybind/scikit_build_example>`_
+• `CMake example <https://github.com/pybind/cmake_example>`_
+
+.. start
+
+
+**pybind11** is a lightweight header-only library that exposes C++ types
+in Python and vice versa, mainly to create Python bindings of existing
+C++ code. Its goals and syntax are similar to the excellent
+`Boost.Python <http://www.boost.org/doc/libs/1_58_0/libs/python/doc/>`_
+library by David Abrahams: to minimize boilerplate code in traditional
+extension modules by inferring type information using compile-time
+introspection.
+
+The main issue with Boost.Python—and the reason for creating such a
+similar project—is Boost. Boost is an enormously large and complex suite
+of utility libraries that works with almost every C++ compiler in
+existence. This compatibility has its cost: arcane template tricks and
+workarounds are necessary to support the oldest and buggiest of compiler
+specimens. Now that C++11-compatible compilers are widely available,
+this heavy machinery has become an excessively large and unnecessary
+dependency.
+
+Think of this library as a tiny self-contained version of Boost.Python
+with everything stripped away that isn't relevant for binding
+generation. Without comments, the core header files only require ~4K
+lines of code and depend on Python (3.6+, or PyPy) and the C++
+standard library. This compact implementation was possible thanks to
+some of the new C++11 language features (specifically: tuples, lambda
+functions and variadic templates). Since its creation, this library has
+grown beyond Boost.Python in many ways, leading to dramatically simpler
+binding code in many common situations.
+
+Tutorial and reference documentation is provided at
+`pybind11.readthedocs.io <https://pybind11.readthedocs.io/en/latest>`_.
+A PDF version of the manual is available
+`here <https://pybind11.readthedocs.io/_/downloads/en/latest/pdf/>`_.
+And the source code is always available at
+`github.com/pybind/pybind11 <https://github.com/pybind/pybind11>`_.
+
+
+Core features
+-------------
+
+
+pybind11 can map the following core C++ features to Python:
+
+- Functions accepting and returning custom data structures per value,
+  reference, or pointer
+- Instance methods and static methods
+- Overloaded functions
+- Instance attributes and static attributes
+- Arbitrary exception types
+- Enumerations
+- Callbacks
+- Iterators and ranges
+- Custom operators
+- Single and multiple inheritance
+- STL data structures
+- Smart pointers with reference counting like ``std::shared_ptr``
+- Internal references with correct reference counting
+- C++ classes with virtual (and pure virtual) methods can be extended
+  in Python
+
+Goodies
+-------
+
+In addition to the core functionality, pybind11 provides some extra
+goodies:
+
+- Python 3.6+, and PyPy3 7.3 are supported with an implementation-agnostic
+  interface (pybind11 2.9 was the last version to support Python 2 and 3.5).
+
+- It is possible to bind C++11 lambda functions with captured
+  variables. The lambda capture data is stored inside the resulting
+  Python function object.
+
+- pybind11 uses C++11 move constructors and move assignment operators
+  whenever possible to efficiently transfer custom data types.
+
+- It's easy to expose the internal storage of custom data types through
+  Pythons' buffer protocols. This is handy e.g. for fast conversion
+  between C++ matrix classes like Eigen and NumPy without expensive
+  copy operations.
+
+- pybind11 can automatically vectorize functions so that they are
+  transparently applied to all entries of one or more NumPy array
+  arguments.
+
+- Python's slice-based access and assignment operations can be
+  supported with just a few lines of code.
+
+- Everything is contained in just a few header files; there is no need
+  to link against any additional libraries.
+
+- Binaries are generally smaller by a factor of at least 2 compared to
+  equivalent bindings generated by Boost.Python. A recent pybind11
+  conversion of PyRosetta, an enormous Boost.Python binding project,
+  `reported <https://graylab.jhu.edu/Sergey/2016.RosettaCon/PyRosetta-4.pdf>`_
+  a binary size reduction of **5.4x** and compile time reduction by
+  **5.8x**.
+
+- Function signatures are precomputed at compile time (using
+  ``constexpr``), leading to smaller binaries.
+
+- With little extra effort, C++ types can be pickled and unpickled
+  similar to regular Python objects.
+
+Supported compilers
+-------------------
+
+1. Clang/LLVM 3.3 or newer (for Apple Xcode's clang, this is 5.0.0 or
+   newer)
+2. GCC 4.8 or newer
+3. Microsoft Visual Studio 2017 or newer
+4. Intel classic C++ compiler 18 or newer (ICC 20.2 tested in CI)
+5. Cygwin/GCC (previously tested on 2.5.1)
+6. NVCC (CUDA 11.0 tested in CI)
+7. NVIDIA PGI (20.9 tested in CI)
+
+About
+-----
+
+This project was created by `Wenzel
+Jakob <http://rgl.epfl.ch/people/wjakob>`_. Significant features and/or
+improvements to the code were contributed by Jonas Adler, Lori A. Burns,
+Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel
+Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz Miąsko,
+Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim
+Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart.
+
+We thank Google for a generous financial contribution to the continuous
+integration infrastructure used by this project.
+
+
+Contributing
+~~~~~~~~~~~~
+
+See the `contributing
+guide <https://github.com/pybind/pybind11/blob/master/.github/CONTRIBUTING.md>`_
+for information on building and contributing to pybind11.
+
+License
+~~~~~~~
+
+pybind11 is provided under a BSD-style license that can be found in the
+`LICENSE <https://github.com/pybind/pybind11/blob/master/LICENSE>`_
+file. By using, distributing, or contributing to this project, you agree
+to the terms and conditions of this license.
+
+.. |Latest Documentation Status| image:: https://readthedocs.org/projects/pybind11/badge?version=latest
+   :target: http://pybind11.readthedocs.org/en/latest
+.. |Stable Documentation Status| image:: https://img.shields.io/badge/docs-stable-blue.svg
+   :target: http://pybind11.readthedocs.org/en/stable
+.. |Gitter chat| image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg
+   :target: https://gitter.im/pybind/Lobby
+.. |CI| image:: https://github.com/pybind/pybind11/workflows/CI/badge.svg
+   :target: https://github.com/pybind/pybind11/actions
+.. |Build status| image:: https://ci.appveyor.com/api/projects/status/riaj54pn4h08xy40?svg=true
+   :target: https://ci.appveyor.com/project/wjakob/pybind11
+.. |PyPI package| image:: https://img.shields.io/pypi/v/pybind11.svg
+   :target: https://pypi.org/project/pybind11/
+.. |Conda-forge| image:: https://img.shields.io/conda/vn/conda-forge/pybind11.svg
+   :target: https://github.com/conda-forge/pybind11-feedstock
+.. |Repology| image:: https://repology.org/badge/latest-versions/python:pybind11.svg
+   :target: https://repology.org/project/python:pybind11/versions
+.. |Python Versions| image:: https://img.shields.io/pypi/pyversions/pybind11.svg
+   :target: https://pypi.org/project/pybind11/
+.. |GitHub Discussions| image:: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github
+   :target: https://github.com/pybind/pybind11/discussions

libs/pybind/docs/Doxyfile

@@ -0,0 +1,21 @@
+PROJECT_NAME           = pybind11
+INPUT                  = ../include/pybind11/
+RECURSIVE              = YES
+
+GENERATE_HTML          = NO
+GENERATE_LATEX         = NO
+GENERATE_XML           = YES
+XML_OUTPUT             = .build/doxygenxml
+XML_PROGRAMLISTING     = YES
+
+MACRO_EXPANSION        = YES
+EXPAND_ONLY_PREDEF     = YES
+EXPAND_AS_DEFINED      = PYBIND11_RUNTIME_EXCEPTION
+
+ALIASES                = "rst=\verbatim embed:rst"
+ALIASES               += "endrst=\endverbatim"
+
+QUIET                  = YES
+WARNINGS               = YES
+WARN_IF_UNDOCUMENTED   = NO
+PREDEFINED             = PYBIND11_NOINLINE

libs/pybind/docs/Makefile

@@ -0,0 +1,192 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = .build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  applehelp  to make an Apple Help Book"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+
+clean:
+	rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pybind11.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pybind11.qhc"
+
+applehelp:
+	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
+	@echo
+	@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
+	@echo "N.B. You won't be able to view it unless you put it in" \
+	      "~/Library/Documentation/Help or install it in your application" \
+	      "bundle."
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/pybind11"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pybind11"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+coverage:
+	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+	@echo "Testing of coverage in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/coverage/python.txt."
+
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

libs/pybind/docs/_static/css/custom.css

@@ -0,0 +1,3 @@
+.highlight .go {
+  color: #707070;
+}

libs/pybind/docs/advanced/cast/chrono.rst

@@ -0,0 +1,81 @@
+Chrono
+======
+
+When including the additional header file :file:`pybind11/chrono.h` conversions
+from C++11 chrono datatypes to python datetime objects are automatically enabled.
+This header also enables conversions of python floats (often from sources such
+as ``time.monotonic()``, ``time.perf_counter()`` and ``time.process_time()``)
+into durations.
+
+An overview of clocks in C++11
+------------------------------
+
+A point of confusion when using these conversions is the differences between
+clocks provided in C++11. There are three clock types defined by the C++11
+standard and users can define their own if needed. Each of these clocks have
+different properties and when converting to and from python will give different
+results.
+
+The first clock defined by the standard is ``std::chrono::system_clock``. This
+clock measures the current date and time. However, this clock changes with to
+updates to the operating system time. For example, if your time is synchronised
+with a time server this clock will change. This makes this clock a poor choice
+for timing purposes but good for measuring the wall time.
+
+The second clock defined in the standard is ``std::chrono::steady_clock``.
+This clock ticks at a steady rate and is never adjusted. This makes it excellent
+for timing purposes, however the value in this clock does not correspond to the
+current date and time. Often this clock will be the amount of time your system
+has been on, although it does not have to be. This clock will never be the same
+clock as the system clock as the system clock can change but steady clocks
+cannot.
+
+The third clock defined in the standard is ``std::chrono::high_resolution_clock``.
+This clock is the clock that has the highest resolution out of the clocks in the
+system. It is normally a typedef to either the system clock or the steady clock
+but can be its own independent clock. This is important as when using these
+conversions as the types you get in python for this clock might be different
+depending on the system.
+If it is a typedef of the system clock, python will get datetime objects, but if
+it is a different clock they will be timedelta objects.
+
+Provided conversions
+--------------------
+
+.. rubric:: C++ to Python
+
+- ``std::chrono::system_clock::time_point`` → ``datetime.datetime``
+    System clock times are converted to python datetime instances. They are
+    in the local timezone, but do not have any timezone information attached
+    to them (they are naive datetime objects).
+
+- ``std::chrono::duration`` → ``datetime.timedelta``
+    Durations are converted to timedeltas, any precision in the duration
+    greater than microseconds is lost by rounding towards zero.
+
+- ``std::chrono::[other_clocks]::time_point`` → ``datetime.timedelta``
+    Any clock time that is not the system clock is converted to a time delta.
+    This timedelta measures the time from the clocks epoch to now.
+
+.. rubric:: Python to C++
+
+- ``datetime.datetime`` or ``datetime.date`` or ``datetime.time`` → ``std::chrono::system_clock::time_point``
+    Date/time objects are converted into system clock timepoints. Any
+    timezone information is ignored and the type is treated as a naive
+    object.
+
+- ``datetime.timedelta`` → ``std::chrono::duration``
+    Time delta are converted into durations with microsecond precision.
+
+- ``datetime.timedelta`` → ``std::chrono::[other_clocks]::time_point``
+    Time deltas that are converted into clock timepoints are treated as
+    the amount of time from the start of the clocks epoch.
+
+- ``float`` → ``std::chrono::duration``
+    Floats that are passed to C++ as durations be interpreted as a number of
+    seconds. These will be converted to the duration using ``duration_cast``
+    from the float.
+
+- ``float`` → ``std::chrono::[other_clocks]::time_point``
+    Floats that are passed to C++ as time points will be interpreted as the
+    number of seconds from the start of the clocks epoch.

libs/pybind/docs/advanced/cast/custom.rst

@@ -0,0 +1,93 @@
+Custom type casters
+===================
+
+In very rare cases, applications may require custom type casters that cannot be
+expressed using the abstractions provided by pybind11, thus requiring raw
+Python C API calls. This is fairly advanced usage and should only be pursued by
+experts who are familiar with the intricacies of Python reference counting.
+
+The following snippets demonstrate how this works for a very simple ``inty``
+type that that should be convertible from Python types that provide a
+``__int__(self)`` method.
+
+.. code-block:: cpp
+
+    struct inty { long long_value; };
+
+    void print(inty s) {
+        std::cout << s.long_value << std::endl;
+    }
+
+The following Python snippet demonstrates the intended usage from the Python side:
+
+.. code-block:: python
+
+    class A:
+        def __int__(self):
+            return 123
+
+
+    from example import print
+
+    print(A())
+
+To register the necessary conversion routines, it is necessary to add an
+instantiation of the ``pybind11::detail::type_caster<T>`` template.
+Although this is an implementation detail, adding an instantiation of this
+type is explicitly allowed.
+
+.. code-block:: cpp
+
+    namespace pybind11 { namespace detail {
+        template <> struct type_caster<inty> {
+        public:
+            /**
+             * This macro establishes the name 'inty' in
+             * function signatures and declares a local variable
+             * 'value' of type inty
+             */
+            PYBIND11_TYPE_CASTER(inty, const_name("inty"));
+
+            /**
+             * Conversion part 1 (Python->C++): convert a PyObject into a inty
+             * instance or return false upon failure. The second argument
+             * indicates whether implicit conversions should be applied.
+             */
+            bool load(handle src, bool) {
+                /* Extract PyObject from handle */
+                PyObject *source = src.ptr();
+                /* Try converting into a Python integer value */
+                PyObject *tmp = PyNumber_Long(source);
+                if (!tmp)
+                    return false;
+                /* Now try to convert into a C++ int */
+                value.long_value = PyLong_AsLong(tmp);
+                Py_DECREF(tmp);
+                /* Ensure return code was OK (to avoid out-of-range errors etc) */
+                return !(value.long_value == -1 && !PyErr_Occurred());
+            }
+
+            /**
+             * Conversion part 2 (C++ -> Python): convert an inty instance into
+             * a Python object. The second and third arguments are used to
+             * indicate the return value policy and parent object (for
+             * ``return_value_policy::reference_internal``) and are generally
+             * ignored by implicit casters.
+             */
+            static handle cast(inty src, return_value_policy /* policy */, handle /* parent */) {
+                return PyLong_FromLong(src.long_value);
+            }
+        };
+    }} // namespace pybind11::detail
+
+.. note::
+
+    A ``type_caster<T>`` defined with ``PYBIND11_TYPE_CASTER(T, ...)`` requires
+    that ``T`` is default-constructible (``value`` is first default constructed
+    and then ``load()`` assigns to it).
+
+.. warning::
+
+    When using custom type casters, it's important to declare them consistently
+    in every compilation unit of the Python extension module. Otherwise,
+    undefined behavior can ensue.

libs/pybind/docs/advanced/cast/eigen.rst

@@ -0,0 +1,310 @@
+Eigen
+#####
+
+`Eigen <http://eigen.tuxfamily.org>`_ is C++ header-based library for dense and
+sparse linear algebra. Due to its popularity and widespread adoption, pybind11
+provides transparent conversion and limited mapping support between Eigen and
+Scientific Python linear algebra data types.
+
+To enable the built-in Eigen support you must include the optional header file
+:file:`pybind11/eigen.h`.
+
+Pass-by-value
+=============
+
+When binding a function with ordinary Eigen dense object arguments (for
+example, ``Eigen::MatrixXd``), pybind11 will accept any input value that is
+already (or convertible to) a ``numpy.ndarray`` with dimensions compatible with
+the Eigen type, copy its values into a temporary Eigen variable of the
+appropriate type, then call the function with this temporary variable.
+
+Sparse matrices are similarly copied to or from
+``scipy.sparse.csr_matrix``/``scipy.sparse.csc_matrix`` objects.
+
+Pass-by-reference
+=================
+
+One major limitation of the above is that every data conversion implicitly
+involves a copy, which can be both expensive (for large matrices) and disallows
+binding functions that change their (Matrix) arguments.  Pybind11 allows you to
+work around this by using Eigen's ``Eigen::Ref<MatrixType>`` class much as you
+would when writing a function taking a generic type in Eigen itself (subject to
+some limitations discussed below).
+
+When calling a bound function accepting a ``Eigen::Ref<const MatrixType>``
+type, pybind11 will attempt to avoid copying by using an ``Eigen::Map`` object
+that maps into the source ``numpy.ndarray`` data: this requires both that the
+data types are the same (e.g. ``dtype='float64'`` and ``MatrixType::Scalar`` is
+``double``); and that the storage is layout compatible.  The latter limitation
+is discussed in detail in the section below, and requires careful
+consideration: by default, numpy matrices and Eigen matrices are *not* storage
+compatible.
+
+If the numpy matrix cannot be used as is (either because its types differ, e.g.
+passing an array of integers to an Eigen parameter requiring doubles, or
+because the storage is incompatible), pybind11 makes a temporary copy and
+passes the copy instead.
+
+When a bound function parameter is instead ``Eigen::Ref<MatrixType>`` (note the
+lack of ``const``), pybind11 will only allow the function to be called if it
+can be mapped *and* if the numpy array is writeable (that is
+``a.flags.writeable`` is true).  Any access (including modification) made to
+the passed variable will be transparently carried out directly on the
+``numpy.ndarray``.
+
+This means you can write code such as the following and have it work as
+expected:
+
+.. code-block:: cpp
+
+    void scale_by_2(Eigen::Ref<Eigen::VectorXd> v) {
+        v *= 2;
+    }
+
+Note, however, that you will likely run into limitations due to numpy and
+Eigen's difference default storage order for data; see the below section on
+:ref:`storage_orders` for details on how to bind code that won't run into such
+limitations.
+
+.. note::
+
+    Passing by reference is not supported for sparse types.
+
+Returning values to Python
+==========================
+
+When returning an ordinary dense Eigen matrix type to numpy (e.g.
+``Eigen::MatrixXd`` or ``Eigen::RowVectorXf``) pybind11 keeps the matrix and
+returns a numpy array that directly references the Eigen matrix: no copy of the
+data is performed.  The numpy array will have ``array.flags.owndata`` set to
+``False`` to indicate that it does not own the data, and the lifetime of the
+stored Eigen matrix will be tied to the returned ``array``.
+
+If you bind a function with a non-reference, ``const`` return type (e.g.
+``const Eigen::MatrixXd``), the same thing happens except that pybind11 also
+sets the numpy array's ``writeable`` flag to false.
+
+If you return an lvalue reference or pointer, the usual pybind11 rules apply,
+as dictated by the binding function's return value policy (see the
+documentation on :ref:`return_value_policies` for full details).  That means,
+without an explicit return value policy, lvalue references will be copied and
+pointers will be managed by pybind11.  In order to avoid copying, you should
+explicitly specify an appropriate return value policy, as in the following
+example:
+
+.. code-block:: cpp
+
+    class MyClass {
+        Eigen::MatrixXd big_mat = Eigen::MatrixXd::Zero(10000, 10000);
+    public:
+        Eigen::MatrixXd &getMatrix() { return big_mat; }
+        const Eigen::MatrixXd &viewMatrix() { return big_mat; }
+    };
+
+    // Later, in binding code:
+    py::class_<MyClass>(m, "MyClass")
+        .def(py::init<>())
+        .def("copy_matrix", &MyClass::getMatrix) // Makes a copy!
+        .def("get_matrix", &MyClass::getMatrix, py::return_value_policy::reference_internal)
+        .def("view_matrix", &MyClass::viewMatrix, py::return_value_policy::reference_internal)
+        ;
+
+.. code-block:: python
+
+    a = MyClass()
+    m = a.get_matrix()  # flags.writeable = True,  flags.owndata = False
+    v = a.view_matrix()  # flags.writeable = False, flags.owndata = False
+    c = a.copy_matrix()  # flags.writeable = True,  flags.owndata = True
+    # m[5,6] and v[5,6] refer to the same element, c[5,6] does not.
+
+Note in this example that ``py::return_value_policy::reference_internal`` is
+used to tie the life of the MyClass object to the life of the returned arrays.
+
+You may also return an ``Eigen::Ref``, ``Eigen::Map`` or other map-like Eigen
+object (for example, the return value of ``matrix.block()`` and related
+methods) that map into a dense Eigen type.  When doing so, the default
+behaviour of pybind11 is to simply reference the returned data: you must take
+care to ensure that this data remains valid!  You may ask pybind11 to
+explicitly *copy* such a return value by using the
+``py::return_value_policy::copy`` policy when binding the function.  You may
+also use ``py::return_value_policy::reference_internal`` or a
+``py::keep_alive`` to ensure the data stays valid as long as the returned numpy
+array does.
+
+When returning such a reference of map, pybind11 additionally respects the
+readonly-status of the returned value, marking the numpy array as non-writeable
+if the reference or map was itself read-only.
+
+.. note::
+
+    Sparse types are always copied when returned.
+
+.. _storage_orders:
+
+Storage orders
+==============
+
+Passing arguments via ``Eigen::Ref`` has some limitations that you must be
+aware of in order to effectively pass matrices by reference.  First and
+foremost is that the default ``Eigen::Ref<MatrixType>`` class requires
+contiguous storage along columns (for column-major types, the default in Eigen)
+or rows if ``MatrixType`` is specifically an ``Eigen::RowMajor`` storage type.
+The former, Eigen's default, is incompatible with ``numpy``'s default row-major
+storage, and so you will not be able to pass numpy arrays to Eigen by reference
+without making one of two changes.
+
+(Note that this does not apply to vectors (or column or row matrices): for such
+types the "row-major" and "column-major" distinction is meaningless).
+
+The first approach is to change the use of ``Eigen::Ref<MatrixType>`` to the
+more general ``Eigen::Ref<MatrixType, 0, Eigen::Stride<Eigen::Dynamic,
+Eigen::Dynamic>>`` (or similar type with a fully dynamic stride type in the
+third template argument).  Since this is a rather cumbersome type, pybind11
+provides a ``py::EigenDRef<MatrixType>`` type alias for your convenience (along
+with EigenDMap for the equivalent Map, and EigenDStride for just the stride
+type).
+
+This type allows Eigen to map into any arbitrary storage order.  This is not
+the default in Eigen for performance reasons: contiguous storage allows
+vectorization that cannot be done when storage is not known to be contiguous at
+compile time.  The default ``Eigen::Ref`` stride type allows non-contiguous
+storage along the outer dimension (that is, the rows of a column-major matrix
+or columns of a row-major matrix), but not along the inner dimension.
+
+This type, however, has the added benefit of also being able to map numpy array
+slices.  For example, the following (contrived) example uses Eigen with a numpy
+slice to multiply by 2 all coefficients that are both on even rows (0, 2, 4,
+...) and in columns 2, 5, or 8:
+
+.. code-block:: cpp
+
+    m.def("scale", [](py::EigenDRef<Eigen::MatrixXd> m, double c) { m *= c; });
+
+.. code-block:: python
+
+    # a = np.array(...)
+    scale_by_2(myarray[0::2, 2:9:3])
+
+The second approach to avoid copying is more intrusive: rearranging the
+underlying data types to not run into the non-contiguous storage problem in the
+first place.  In particular, that means using matrices with ``Eigen::RowMajor``
+storage, where appropriate, such as:
+
+.. code-block:: cpp
+
+    using RowMatrixXd = Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
+    // Use RowMatrixXd instead of MatrixXd
+
+Now bound functions accepting ``Eigen::Ref<RowMatrixXd>`` arguments will be
+callable with numpy's (default) arrays without involving a copying.
+
+You can, alternatively, change the storage order that numpy arrays use by
+adding the ``order='F'`` option when creating an array:
+
+.. code-block:: python
+
+    myarray = np.array(source, order="F")
+
+Such an object will be passable to a bound function accepting an
+``Eigen::Ref<MatrixXd>`` (or similar column-major Eigen type).
+
+One major caveat with this approach, however, is that it is not entirely as
+easy as simply flipping all Eigen or numpy usage from one to the other: some
+operations may alter the storage order of a numpy array.  For example, ``a2 =
+array.transpose()`` results in ``a2`` being a view of ``array`` that references
+the same data, but in the opposite storage order!
+
+While this approach allows fully optimized vectorized calculations in Eigen, it
+cannot be used with array slices, unlike the first approach.
+
+When *returning* a matrix to Python (either a regular matrix, a reference via
+``Eigen::Ref<>``, or a map/block into a matrix), no special storage
+consideration is required: the created numpy array will have the required
+stride that allows numpy to properly interpret the array, whatever its storage
+order.
+
+Failing rather than copying
+===========================
+
+The default behaviour when binding ``Eigen::Ref<const MatrixType>`` Eigen
+references is to copy matrix values when passed a numpy array that does not
+conform to the element type of ``MatrixType`` or does not have a compatible
+stride layout.  If you want to explicitly avoid copying in such a case, you
+should bind arguments using the ``py::arg().noconvert()`` annotation (as
+described in the :ref:`nonconverting_arguments` documentation).
+
+The following example shows an example of arguments that don't allow data
+copying to take place:
+
+.. code-block:: cpp
+
+    // The method and function to be bound:
+    class MyClass {
+        // ...
+        double some_method(const Eigen::Ref<const MatrixXd> &matrix) { /* ... */ }
+    };
+    float some_function(const Eigen::Ref<const MatrixXf> &big,
+                        const Eigen::Ref<const MatrixXf> &small) {
+        // ...
+    }
+
+    // The associated binding code:
+    using namespace pybind11::literals; // for "arg"_a
+    py::class_<MyClass>(m, "MyClass")
+        // ... other class definitions
+        .def("some_method", &MyClass::some_method, py::arg().noconvert());
+
+    m.def("some_function", &some_function,
+        "big"_a.noconvert(), // <- Don't allow copying for this arg
+        "small"_a            // <- This one can be copied if needed
+    );
+
+With the above binding code, attempting to call the the ``some_method(m)``
+method on a ``MyClass`` object, or attempting to call ``some_function(m, m2)``
+will raise a ``RuntimeError`` rather than making a temporary copy of the array.
+It will, however, allow the ``m2`` argument to be copied into a temporary if
+necessary.
+
+Note that explicitly specifying ``.noconvert()`` is not required for *mutable*
+Eigen references (e.g. ``Eigen::Ref<MatrixXd>`` without ``const`` on the
+``MatrixXd``): mutable references will never be called with a temporary copy.
+
+Vectors versus column/row matrices
+==================================
+
+Eigen and numpy have fundamentally different notions of a vector.  In Eigen, a
+vector is simply a matrix with the number of columns or rows set to 1 at
+compile time (for a column vector or row vector, respectively).  NumPy, in
+contrast, has comparable 2-dimensional 1xN and Nx1 arrays, but *also* has
+1-dimensional arrays of size N.
+
+When passing a 2-dimensional 1xN or Nx1 array to Eigen, the Eigen type must
+have matching dimensions: That is, you cannot pass a 2-dimensional Nx1 numpy
+array to an Eigen value expecting a row vector, or a 1xN numpy array as a
+column vector argument.
+
+On the other hand, pybind11 allows you to pass 1-dimensional arrays of length N
+as Eigen parameters.  If the Eigen type can hold a column vector of length N it
+will be passed as such a column vector.  If not, but the Eigen type constraints
+will accept a row vector, it will be passed as a row vector.  (The column
+vector takes precedence when both are supported, for example, when passing a
+1D numpy array to a MatrixXd argument).  Note that the type need not be
+explicitly a vector: it is permitted to pass a 1D numpy array of size 5 to an
+Eigen ``Matrix<double, Dynamic, 5>``: you would end up with a 1x5 Eigen matrix.
+Passing the same to an ``Eigen::MatrixXd`` would result in a 5x1 Eigen matrix.
+
+When returning an Eigen vector to numpy, the conversion is ambiguous: a row
+vector of length 4 could be returned as either a 1D array of length 4, or as a
+2D array of size 1x4.  When encountering such a situation, pybind11 compromises
+by considering the returned Eigen type: if it is a compile-time vector--that
+is, the type has either the number of rows or columns set to 1 at compile
+time--pybind11 converts to a 1D numpy array when returning the value.  For
+instances that are a vector only at run-time (e.g. ``MatrixXd``,
+``Matrix<float, Dynamic, 4>``), pybind11 returns the vector as a 2D array to
+numpy.  If this isn't want you want, you can use ``array.reshape(...)`` to get
+a view of the same data in the desired dimensions.
+
+.. seealso::
+
+    The file :file:`tests/test_eigen.cpp` contains a complete example that
+    shows how to pass Eigen sparse and dense data types in more detail.

libs/pybind/docs/advanced/cast/functional.rst

@@ -0,0 +1,109 @@
+Functional
+##########
+
+The following features must be enabled by including :file:`pybind11/functional.h`.
+
+
+Callbacks and passing anonymous functions
+=========================================
+
+The C++11 standard brought lambda functions and the generic polymorphic
+function wrapper ``std::function<>`` to the C++ programming language, which
+enable powerful new ways of working with functions. Lambda functions come in
+two flavors: stateless lambda function resemble classic function pointers that
+link to an anonymous piece of code, while stateful lambda functions
+additionally depend on captured variables that are stored in an anonymous
+*lambda closure object*.
+
+Here is a simple example of a C++ function that takes an arbitrary function
+(stateful or stateless) with signature ``int -> int`` as an argument and runs
+it with the value 10.
+
+.. code-block:: cpp
+
+    int func_arg(const std::function<int(int)> &f) {
+        return f(10);
+    }
+
+The example below is more involved: it takes a function of signature ``int -> int``
+and returns another function of the same kind. The return value is a stateful
+lambda function, which stores the value ``f`` in the capture object and adds 1 to
+its return value upon execution.
+
+.. code-block:: cpp
+
+    std::function<int(int)> func_ret(const std::function<int(int)> &f) {
+        return [f](int i) {
+            return f(i) + 1;
+        };
+    }
+
+This example demonstrates using python named parameters in C++ callbacks which
+requires using ``py::cpp_function`` as a wrapper. Usage is similar to defining
+methods of classes:
+
+.. code-block:: cpp
+
+    py::cpp_function func_cpp() {
+        return py::cpp_function([](int i) { return i+1; },
+           py::arg("number"));
+    }
+
+After including the extra header file :file:`pybind11/functional.h`, it is almost
+trivial to generate binding code for all of these functions.
+
+.. code-block:: cpp
+
+    #include <pybind11/functional.h>
+
+    PYBIND11_MODULE(example, m) {
+        m.def("func_arg", &func_arg);
+        m.def("func_ret", &func_ret);
+        m.def("func_cpp", &func_cpp);
+    }
+
+The following interactive session shows how to call them from Python.
+
+.. code-block:: pycon
+
+    $ python
+    >>> import example
+    >>> def square(i):
+    ...     return i * i
+    ...
+    >>> example.func_arg(square)
+    100L
+    >>> square_plus_1 = example.func_ret(square)
+    >>> square_plus_1(4)
+    17L
+    >>> plus_1 = func_cpp()
+    >>> plus_1(number=43)
+    44L
+
+.. warning::
+
+    Keep in mind that passing a function from C++ to Python (or vice versa)
+    will instantiate a piece of wrapper code that translates function
+    invocations between the two languages. Naturally, this translation
+    increases the computational cost of each function call somewhat. A
+    problematic situation can arise when a function is copied back and forth
+    between Python and C++ many times in a row, in which case the underlying
+    wrappers will accumulate correspondingly. The resulting long sequence of
+    C++ -> Python -> C++ -> ... roundtrips can significantly decrease
+    performance.
+
+    There is one exception: pybind11 detects case where a stateless function
+    (i.e. a function pointer or a lambda function without captured variables)
+    is passed as an argument to another C++ function exposed in Python. In this
+    case, there is no overhead. Pybind11 will extract the underlying C++
+    function pointer from the wrapped function to sidestep a potential C++ ->
+    Python -> C++ roundtrip. This is demonstrated in :file:`tests/test_callbacks.cpp`.
+
+.. note::
+
+    This functionality is very useful when generating bindings for callbacks in
+    C++ libraries (e.g. GUI libraries, asynchronous networking libraries, etc.).
+
+    The file :file:`tests/test_callbacks.cpp` contains a complete example
+    that demonstrates how to work with callbacks and anonymous functions in
+    more detail.

libs/pybind/docs/advanced/cast/index.rst

@@ -0,0 +1,43 @@
+.. _type-conversions:
+
+Type conversions
+################
+
+Apart from enabling cross-language function calls, a fundamental problem
+that a binding tool like pybind11 must address is to provide access to
+native Python types in C++ and vice versa. There are three fundamentally
+different ways to do this—which approach is preferable for a particular type
+depends on the situation at hand.
+
+1. Use a native C++ type everywhere. In this case, the type must be wrapped
+   using pybind11-generated bindings so that Python can interact with it.
+
+2. Use a native Python type everywhere. It will need to be wrapped so that
+   C++ functions can interact with it.
+
+3. Use a native C++ type on the C++ side and a native Python type on the
+   Python side. pybind11 refers to this as a *type conversion*.
+
+   Type conversions are the most "natural" option in the sense that native
+   (non-wrapped) types are used everywhere. The main downside is that a copy
+   of the data must be made on every Python ↔ C++ transition: this is
+   needed since the C++ and Python versions of the same type generally won't
+   have the same memory layout.
+
+   pybind11 can perform many kinds of conversions automatically. An overview
+   is provided in the table ":ref:`conversion_table`".
+
+The following subsections discuss the differences between these options in more
+detail. The main focus in this section is on type conversions, which represent
+the last case of the above list.
+
+.. toctree::
+   :maxdepth: 1
+
+   overview
+   strings
+   stl
+   functional
+   chrono
+   eigen
+   custom

libs/pybind/docs/advanced/cast/overview.rst

@@ -0,0 +1,170 @@
+Overview
+########
+
+.. rubric:: 1. Native type in C++, wrapper in Python
+
+Exposing a custom C++ type using :class:`py::class_` was covered in detail
+in the :doc:`/classes` section. There, the underlying data structure is
+always the original C++ class while the :class:`py::class_` wrapper provides
+a Python interface. Internally, when an object like this is sent from C++ to
+Python, pybind11 will just add the outer wrapper layer over the native C++
+object. Getting it back from Python is just a matter of peeling off the
+wrapper.
+
+.. rubric:: 2. Wrapper in C++, native type in Python
+
+This is the exact opposite situation. Now, we have a type which is native to
+Python, like a ``tuple`` or a ``list``. One way to get this data into C++ is
+with the :class:`py::object` family of wrappers. These are explained in more
+detail in the :doc:`/advanced/pycpp/object` section. We'll just give a quick
+example here:
+
+.. code-block:: cpp
+
+    void print_list(py::list my_list) {
+        for (auto item : my_list)
+            std::cout << item << " ";
+    }
+
+.. code-block:: pycon
+
+    >>> print_list([1, 2, 3])
+    1 2 3
+
+The Python ``list`` is not converted in any way -- it's just wrapped in a C++
+:class:`py::list` class. At its core it's still a Python object. Copying a
+:class:`py::list` will do the usual reference-counting like in Python.
+Returning the object to Python will just remove the thin wrapper.
+
+.. rubric:: 3. Converting between native C++ and Python types
+
+In the previous two cases we had a native type in one language and a wrapper in
+the other. Now, we have native types on both sides and we convert between them.
+
+.. code-block:: cpp
+
+    void print_vector(const std::vector<int> &v) {
+        for (auto item : v)
+            std::cout << item << "\n";
+    }
+
+.. code-block:: pycon
+
+    >>> print_vector([1, 2, 3])
+    1 2 3
+
+In this case, pybind11 will construct a new ``std::vector<int>`` and copy each
+element from the Python ``list``. The newly constructed object will be passed
+to ``print_vector``. The same thing happens in the other direction: a new
+``list`` is made to match the value returned from C++.
+
+Lots of these conversions are supported out of the box, as shown in the table
+below. They are very convenient, but keep in mind that these conversions are
+fundamentally based on copying data. This is perfectly fine for small immutable
+types but it may become quite expensive for large data structures. This can be
+avoided by overriding the automatic conversion with a custom wrapper (i.e. the
+above-mentioned approach 1). This requires some manual effort and more details
+are available in the :ref:`opaque` section.
+
+.. _conversion_table:
+
+List of all builtin conversions
+-------------------------------
+
+The following basic data types are supported out of the box (some may require
+an additional extension header to be included). To pass other data structures
+as arguments and return values, refer to the section on binding :ref:`classes`.
+
++------------------------------------+---------------------------+-----------------------------------+
+|  Data type                         |  Description              | Header file                       |
++====================================+===========================+===================================+
+| ``int8_t``, ``uint8_t``            | 8-bit integers            | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``int16_t``, ``uint16_t``          | 16-bit integers           | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``int32_t``, ``uint32_t``          | 32-bit integers           | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``int64_t``, ``uint64_t``          | 64-bit integers           | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``ssize_t``, ``size_t``            | Platform-dependent size   | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``float``, ``double``              | Floating point types      | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``bool``                           | Two-state Boolean type    | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``char``                           | Character literal         | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``char16_t``                       | UTF-16 character literal  | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``char32_t``                       | UTF-32 character literal  | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``wchar_t``                        | Wide character literal    | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``const char *``                   | UTF-8 string literal      | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``const char16_t *``               | UTF-16 string literal     | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``const char32_t *``               | UTF-32 string literal     | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``const wchar_t *``                | Wide string literal       | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::string``                    | STL dynamic UTF-8 string  | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::u16string``                 | STL dynamic UTF-16 string | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::u32string``                 | STL dynamic UTF-32 string | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::wstring``                   | STL dynamic wide string   | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::string_view``,              | STL C++17 string views    | :file:`pybind11/pybind11.h`       |
+| ``std::u16string_view``, etc.      |                           |                                   |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::pair<T1, T2>``              | Pair of two custom types  | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::tuple<...>``                | Arbitrary tuple of types  | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::reference_wrapper<...>``    | Reference type wrapper    | :file:`pybind11/pybind11.h`       |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::complex<T>``                | Complex numbers           | :file:`pybind11/complex.h`        |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::array<T, Size>``            | STL static array          | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::vector<T>``                 | STL dynamic array         | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::deque<T>``                  | STL double-ended queue    | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::valarray<T>``               | STL value array           | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::list<T>``                   | STL linked list           | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::map<T1, T2>``               | STL ordered map           | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::unordered_map<T1, T2>``     | STL unordered map         | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::set<T>``                    | STL ordered set           | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::unordered_set<T>``          | STL unordered set         | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::optional<T>``               | STL optional type (C++17) | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::experimental::optional<T>`` | STL optional type (exp.)  | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::variant<...>``              | Type-safe union (C++17)   | :file:`pybind11/stl.h`            |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::filesystem::path<T>``       | STL path (C++17) [#]_     | :file:`pybind11/stl/filesystem.h` |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::function<...>``             | STL polymorphic function  | :file:`pybind11/functional.h`     |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::chrono::duration<...>``     | STL time duration         | :file:`pybind11/chrono.h`         |
++------------------------------------+---------------------------+-----------------------------------+
+| ``std::chrono::time_point<...>``   | STL date/time             | :file:`pybind11/chrono.h`         |
++------------------------------------+---------------------------+-----------------------------------+
+| ``Eigen::Matrix<...>``             | Eigen: dense matrix       | :file:`pybind11/eigen.h`          |
++------------------------------------+---------------------------+-----------------------------------+
+| ``Eigen::Map<...>``                | Eigen: mapped memory      | :file:`pybind11/eigen.h`          |
++------------------------------------+---------------------------+-----------------------------------+
+| ``Eigen::SparseMatrix<...>``       | Eigen: sparse matrix      | :file:`pybind11/eigen.h`          |
++------------------------------------+---------------------------+-----------------------------------+
+
+.. [#] ``std::filesystem::path`` is converted to ``pathlib.Path`` and
+   ``os.PathLike`` is converted to ``std::filesystem::path``.

libs/pybind/docs/advanced/cast/stl.rst

@@ -0,0 +1,249 @@
+STL containers
+##############
+
+Automatic conversion
+====================
+
+When including the additional header file :file:`pybind11/stl.h`, conversions
+between ``std::vector<>``/``std::deque<>``/``std::list<>``/``std::array<>``/``std::valarray<>``,
+``std::set<>``/``std::unordered_set<>``, and
+``std::map<>``/``std::unordered_map<>`` and the Python ``list``, ``set`` and
+``dict`` data structures are automatically enabled. The types ``std::pair<>``
+and ``std::tuple<>`` are already supported out of the box with just the core
+:file:`pybind11/pybind11.h` header.
+
+The major downside of these implicit conversions is that containers must be
+converted (i.e. copied) on every Python->C++ and C++->Python transition, which
+can have implications on the program semantics and performance. Please read the
+next sections for more details and alternative approaches that avoid this.
+
+.. note::
+
+    Arbitrary nesting of any of these types is possible.
+
+.. seealso::
+
+    The file :file:`tests/test_stl.cpp` contains a complete
+    example that demonstrates how to pass STL data types in more detail.
+
+.. _cpp17_container_casters:
+
+C++17 library containers
+========================
+
+The :file:`pybind11/stl.h` header also includes support for ``std::optional<>``
+and ``std::variant<>``. These require a C++17 compiler and standard library.
+In C++14 mode, ``std::experimental::optional<>`` is supported if available.
+
+Various versions of these containers also exist for C++11 (e.g. in Boost).
+pybind11 provides an easy way to specialize the ``type_caster`` for such
+types:
+
+.. code-block:: cpp
+
+    // `boost::optional` as an example -- can be any `std::optional`-like container
+    namespace pybind11 { namespace detail {
+        template <typename T>
+        struct type_caster<boost::optional<T>> : optional_caster<boost::optional<T>> {};
+    }}
+
+The above should be placed in a header file and included in all translation units
+where automatic conversion is needed. Similarly, a specialization can be provided
+for custom variant types:
+
+.. code-block:: cpp
+
+    // `boost::variant` as an example -- can be any `std::variant`-like container
+    namespace pybind11 { namespace detail {
+        template <typename... Ts>
+        struct type_caster<boost::variant<Ts...>> : variant_caster<boost::variant<Ts...>> {};
+
+        // Specifies the function used to visit the variant -- `apply_visitor` instead of `visit`
+        template <>
+        struct visit_helper<boost::variant> {
+            template <typename... Args>
+            static auto call(Args &&...args) -> decltype(boost::apply_visitor(args...)) {
+                return boost::apply_visitor(args...);
+            }
+        };
+    }} // namespace pybind11::detail
+
+The ``visit_helper`` specialization is not required if your ``name::variant`` provides
+a ``name::visit()`` function. For any other function name, the specialization must be
+included to tell pybind11 how to visit the variant.
+
+.. warning::
+
+    When converting a ``variant`` type, pybind11 follows the same rules as when
+    determining which function overload to call (:ref:`overload_resolution`), and
+    so the same caveats hold. In particular, the order in which the ``variant``'s
+    alternatives are listed is important, since pybind11 will try conversions in
+    this order. This means that, for example, when converting ``variant<int, bool>``,
+    the ``bool`` variant will never be selected, as any Python ``bool`` is already
+    an ``int`` and is convertible to a C++ ``int``. Changing the order of alternatives
+    (and using ``variant<bool, int>``, in this example) provides a solution.
+
+.. note::
+
+    pybind11 only supports the modern implementation of ``boost::variant``
+    which makes use of variadic templates. This requires Boost 1.56 or newer.
+
+.. _opaque:
+
+Making opaque types
+===================
+
+pybind11 heavily relies on a template matching mechanism to convert parameters
+and return values that are constructed from STL data types such as vectors,
+linked lists, hash tables, etc. This even works in a recursive manner, for
+instance to deal with lists of hash maps of pairs of elementary and custom
+types, etc.
+
+However, a fundamental limitation of this approach is that internal conversions
+between Python and C++ types involve a copy operation that prevents
+pass-by-reference semantics. What does this mean?
+
+Suppose we bind the following function
+
+.. code-block:: cpp
+
+    void append_1(std::vector<int> &v) {
+       v.push_back(1);
+    }
+
+and call it from Python, the following happens:
+
+.. code-block:: pycon
+
+   >>> v = [5, 6]
+   >>> append_1(v)
+   >>> print(v)
+   [5, 6]
+
+As you can see, when passing STL data structures by reference, modifications
+are not propagated back the Python side. A similar situation arises when
+exposing STL data structures using the ``def_readwrite`` or ``def_readonly``
+functions:
+
+.. code-block:: cpp
+
+    /* ... definition ... */
+
+    class MyClass {
+        std::vector<int> contents;
+    };
+
+    /* ... binding code ... */
+
+    py::class_<MyClass>(m, "MyClass")
+        .def(py::init<>())
+        .def_readwrite("contents", &MyClass::contents);
+
+In this case, properties can be read and written in their entirety. However, an
+``append`` operation involving such a list type has no effect:
+
+.. code-block:: pycon
+
+   >>> m = MyClass()
+   >>> m.contents = [5, 6]
+   >>> print(m.contents)
+   [5, 6]
+   >>> m.contents.append(7)
+   >>> print(m.contents)
+   [5, 6]
+
+Finally, the involved copy operations can be costly when dealing with very
+large lists. To deal with all of the above situations, pybind11 provides a
+macro named ``PYBIND11_MAKE_OPAQUE(T)`` that disables the template-based
+conversion machinery of types, thus rendering them *opaque*. The contents of
+opaque objects are never inspected or extracted, hence they *can* be passed by
+reference. For instance, to turn ``std::vector<int>`` into an opaque type, add
+the declaration
+
+.. code-block:: cpp
+
+    PYBIND11_MAKE_OPAQUE(std::vector<int>);
+
+before any binding code (e.g. invocations to ``class_::def()``, etc.). This
+macro must be specified at the top level (and outside of any namespaces), since
+it adds a template instantiation of ``type_caster``. If your binding code consists of
+multiple compilation units, it must be present in every file (typically via a
+common header) preceding any usage of ``std::vector<int>``. Opaque types must
+also have a corresponding ``class_`` declaration to associate them with a name
+in Python, and to define a set of available operations, e.g.:
+
+.. code-block:: cpp
+
+    py::class_<std::vector<int>>(m, "IntVector")
+        .def(py::init<>())
+        .def("clear", &std::vector<int>::clear)
+        .def("pop_back", &std::vector<int>::pop_back)
+        .def("__len__", [](const std::vector<int> &v) { return v.size(); })
+        .def("__iter__", [](std::vector<int> &v) {
+           return py::make_iterator(v.begin(), v.end());
+        }, py::keep_alive<0, 1>()) /* Keep vector alive while iterator is used */
+        // ....
+
+.. seealso::
+
+    The file :file:`tests/test_opaque_types.cpp` contains a complete
+    example that demonstrates how to create and expose opaque types using
+    pybind11 in more detail.
+
+.. _stl_bind:
+
+Binding STL containers
+======================
+
+The ability to expose STL containers as native Python objects is a fairly
+common request, hence pybind11 also provides an optional header file named
+:file:`pybind11/stl_bind.h` that does exactly this. The mapped containers try
+to match the behavior of their native Python counterparts as much as possible.
+
+The following example showcases usage of :file:`pybind11/stl_bind.h`:
+
+.. code-block:: cpp
+
+    // Don't forget this
+    #include <pybind11/stl_bind.h>
+
+    PYBIND11_MAKE_OPAQUE(std::vector<int>);
+    PYBIND11_MAKE_OPAQUE(std::map<std::string, double>);
+
+    // ...
+
+    // later in binding code:
+    py::bind_vector<std::vector<int>>(m, "VectorInt");
+    py::bind_map<std::map<std::string, double>>(m, "MapStringDouble");
+
+When binding STL containers pybind11 considers the types of the container's
+elements to decide whether the container should be confined to the local module
+(via the :ref:`module_local` feature).  If the container element types are
+anything other than already-bound custom types bound without
+``py::module_local()`` the container binding will have ``py::module_local()``
+applied.  This includes converting types such as numeric types, strings, Eigen
+types; and types that have not yet been bound at the time of the stl container
+binding.  This module-local binding is designed to avoid potential conflicts
+between module bindings (for example, from two separate modules each attempting
+to bind ``std::vector<int>`` as a python type).
+
+It is possible to override this behavior to force a definition to be either
+module-local or global.  To do so, you can pass the attributes
+``py::module_local()`` (to make the binding module-local) or
+``py::module_local(false)`` (to make the binding global) into the
+``py::bind_vector`` or ``py::bind_map`` arguments:
+
+.. code-block:: cpp
+
+    py::bind_vector<std::vector<int>>(m, "VectorInt", py::module_local(false));
+
+Note, however, that such a global binding would make it impossible to load this
+module at the same time as any other pybind module that also attempts to bind
+the same container type (``std::vector<int>`` in the above example).
+
+See :ref:`module_local` for more details on module-local bindings.
+
+.. seealso::
+
+    The file :file:`tests/test_stl_binders.cpp` shows how to use the
+    convenience STL container wrappers.

libs/pybind/docs/advanced/cast/strings.rst

@@ -0,0 +1,292 @@
+Strings, bytes and Unicode conversions
+######################################
+
+Passing Python strings to C++
+=============================
+
+When a Python ``str`` is passed from Python to a C++ function that accepts
+``std::string`` or ``char *`` as arguments, pybind11 will encode the Python
+string to UTF-8. All Python ``str`` can be encoded in UTF-8, so this operation
+does not fail.
+
+The C++ language is encoding agnostic. It is the responsibility of the
+programmer to track encodings. It's often easiest to simply `use UTF-8
+everywhere <http://utf8everywhere.org/>`_.
+
+.. code-block:: c++
+
+    m.def("utf8_test",
+        [](const std::string &s) {
+            cout << "utf-8 is icing on the cake.\n";
+            cout << s;
+        }
+    );
+    m.def("utf8_charptr",
+        [](const char *s) {
+            cout << "My favorite food is\n";
+            cout << s;
+        }
+    );
+
+.. code-block:: pycon
+
+    >>> utf8_test("🎂")
+    utf-8 is icing on the cake.
+    🎂
+
+    >>> utf8_charptr("🍕")
+    My favorite food is
+    🍕
+
+.. note::
+
+    Some terminal emulators do not support UTF-8 or emoji fonts and may not
+    display the example above correctly.
+
+The results are the same whether the C++ function accepts arguments by value or
+reference, and whether or not ``const`` is used.
+
+Passing bytes to C++
+--------------------
+
+A Python ``bytes`` object will be passed to C++ functions that accept
+``std::string`` or ``char*`` *without* conversion.  In order to make a function
+*only* accept ``bytes`` (and not ``str``), declare it as taking a ``py::bytes``
+argument.
+
+
+Returning C++ strings to Python
+===============================
+
+When a C++ function returns a ``std::string`` or ``char*`` to a Python caller,
+**pybind11 will assume that the string is valid UTF-8** and will decode it to a
+native Python ``str``, using the same API as Python uses to perform
+``bytes.decode('utf-8')``. If this implicit conversion fails, pybind11 will
+raise a ``UnicodeDecodeError``.
+
+.. code-block:: c++
+
+    m.def("std_string_return",
+        []() {
+            return std::string("This string needs to be UTF-8 encoded");
+        }
+    );
+
+.. code-block:: pycon
+
+    >>> isinstance(example.std_string_return(), str)
+    True
+
+
+Because UTF-8 is inclusive of pure ASCII, there is never any issue with
+returning a pure ASCII string to Python. If there is any possibility that the
+string is not pure ASCII, it is necessary to ensure the encoding is valid
+UTF-8.
+
+.. warning::
+
+    Implicit conversion assumes that a returned ``char *`` is null-terminated.
+    If there is no null terminator a buffer overrun will occur.
+
+Explicit conversions
+--------------------
+
+If some C++ code constructs a ``std::string`` that is not a UTF-8 string, one
+can perform a explicit conversion and return a ``py::str`` object. Explicit
+conversion has the same overhead as implicit conversion.
+
+.. code-block:: c++
+
+    // This uses the Python C API to convert Latin-1 to Unicode
+    m.def("str_output",
+        []() {
+            std::string s = "Send your r\xe9sum\xe9 to Alice in HR"; // Latin-1
+            py::str py_s = PyUnicode_DecodeLatin1(s.data(), s.length());
+            return py_s;
+        }
+    );
+
+.. code-block:: pycon
+
+    >>> str_output()
+    'Send your résumé to Alice in HR'
+
+The `Python C API
+<https://docs.python.org/3/c-api/unicode.html#built-in-codecs>`_ provides
+several built-in codecs.
+
+
+One could also use a third party encoding library such as libiconv to transcode
+to UTF-8.
+
+Return C++ strings without conversion
+-------------------------------------
+
+If the data in a C++ ``std::string`` does not represent text and should be
+returned to Python as ``bytes``, then one can return the data as a
+``py::bytes`` object.
+
+.. code-block:: c++
+
+    m.def("return_bytes",
+        []() {
+            std::string s("\xba\xd0\xba\xd0");  // Not valid UTF-8
+            return py::bytes(s);  // Return the data without transcoding
+        }
+    );
+
+.. code-block:: pycon
+
+    >>> example.return_bytes()
+    b'\xba\xd0\xba\xd0'
+
+
+Note the asymmetry: pybind11 will convert ``bytes`` to ``std::string`` without
+encoding, but cannot convert ``std::string`` back to ``bytes`` implicitly.
+
+.. code-block:: c++
+
+    m.def("asymmetry",
+        [](std::string s) {  // Accepts str or bytes from Python
+            return s;  // Looks harmless, but implicitly converts to str
+        }
+    );
+
+.. code-block:: pycon
+
+    >>> isinstance(example.asymmetry(b"have some bytes"), str)
+    True
+
+    >>> example.asymmetry(b"\xba\xd0\xba\xd0")  # invalid utf-8 as bytes
+    UnicodeDecodeError: 'utf-8' codec can't decode byte 0xba in position 0: invalid start byte
+
+
+Wide character strings
+======================
+
+When a Python ``str`` is passed to a C++ function expecting ``std::wstring``,
+``wchar_t*``, ``std::u16string`` or ``std::u32string``, the ``str`` will be
+encoded to UTF-16 or UTF-32 depending on how the C++ compiler implements each
+type, in the platform's native endianness. When strings of these types are
+returned, they are assumed to contain valid UTF-16 or UTF-32, and will be
+decoded to Python ``str``.
+
+.. code-block:: c++
+
+    #define UNICODE
+    #include <windows.h>
+
+    m.def("set_window_text",
+        [](HWND hwnd, std::wstring s) {
+            // Call SetWindowText with null-terminated UTF-16 string
+            ::SetWindowText(hwnd, s.c_str());
+        }
+    );
+    m.def("get_window_text",
+        [](HWND hwnd) {
+            const int buffer_size = ::GetWindowTextLength(hwnd) + 1;
+            auto buffer = std::make_unique< wchar_t[] >(buffer_size);
+
+            ::GetWindowText(hwnd, buffer.data(), buffer_size);
+
+            std::wstring text(buffer.get());
+
+            // wstring will be converted to Python str
+            return text;
+        }
+    );
+
+Strings in multibyte encodings such as Shift-JIS must transcoded to a
+UTF-8/16/32 before being returned to Python.
+
+
+Character literals
+==================
+
+C++ functions that accept character literals as input will receive the first
+character of a Python ``str`` as their input. If the string is longer than one
+Unicode character, trailing characters will be ignored.
+
+When a character literal is returned from C++ (such as a ``char`` or a
+``wchar_t``), it will be converted to a ``str`` that represents the single
+character.
+
+.. code-block:: c++
+
+    m.def("pass_char", [](char c) { return c; });
+    m.def("pass_wchar", [](wchar_t w) { return w; });
+
+.. code-block:: pycon
+
+    >>> example.pass_char("A")
+    'A'
+
+While C++ will cast integers to character types (``char c = 0x65;``), pybind11
+does not convert Python integers to characters implicitly. The Python function
+``chr()`` can be used to convert integers to characters.
+
+.. code-block:: pycon
+
+    >>> example.pass_char(0x65)
+    TypeError
+
+    >>> example.pass_char(chr(0x65))
+    'A'
+
+If the desire is to work with an 8-bit integer, use ``int8_t`` or ``uint8_t``
+as the argument type.
+
+Grapheme clusters
+-----------------
+
+A single grapheme may be represented by two or more Unicode characters. For
+example 'é' is usually represented as U+00E9 but can also be expressed as the
+combining character sequence U+0065 U+0301 (that is, the letter 'e' followed by
+a combining acute accent). The combining character will be lost if the
+two-character sequence is passed as an argument, even though it renders as a
+single grapheme.
+
+.. code-block:: pycon
+
+    >>> example.pass_wchar("é")
+    'é'
+
+    >>> combining_e_acute = "e" + "\u0301"
+
+    >>> combining_e_acute
+    'é'
+
+    >>> combining_e_acute == "é"
+    False
+
+    >>> example.pass_wchar(combining_e_acute)
+    'e'
+
+Normalizing combining characters before passing the character literal to C++
+may resolve *some* of these issues:
+
+.. code-block:: pycon
+
+    >>> example.pass_wchar(unicodedata.normalize("NFC", combining_e_acute))
+    'é'
+
+In some languages (Thai for example), there are `graphemes that cannot be
+expressed as a single Unicode code point
+<http://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries>`_, so there is
+no way to capture them in a C++ character type.
+
+
+C++17 string views
+==================
+
+C++17 string views are automatically supported when compiling in C++17 mode.
+They follow the same rules for encoding and decoding as the corresponding STL
+string type (for example, a ``std::u16string_view`` argument will be passed
+UTF-16-encoded data, and a returned ``std::string_view`` will be decoded as
+UTF-8).
+
+References
+==========
+
+* `The Absolute Minimum Every Software Developer Absolutely, Positively Must Know About Unicode and Character Sets (No Excuses!) <https://www.joelonsoftware.com/2003/10/08/the-absolute-minimum-every-software-developer-absolutely-positively-must-know-about-unicode-and-character-sets-no-excuses/>`_
+* `C++ - Using STL Strings at Win32 API Boundaries <https://msdn.microsoft.com/en-ca/magazine/mt238407.aspx>`_

libs/pybind/docs/advanced/embedding.rst

@@ -0,0 +1,262 @@
+.. _embedding:
+
+Embedding the interpreter
+#########################
+
+While pybind11 is mainly focused on extending Python using C++, it's also
+possible to do the reverse: embed the Python interpreter into a C++ program.
+All of the other documentation pages still apply here, so refer to them for
+general pybind11 usage. This section will cover a few extra things required
+for embedding.
+
+Getting started
+===============
+
+A basic executable with an embedded interpreter can be created with just a few
+lines of CMake and the ``pybind11::embed`` target, as shown below. For more
+information, see :doc:`/compiling`.
+
+.. code-block:: cmake
+
+    cmake_minimum_required(VERSION 3.4)
+    project(example)
+
+    find_package(pybind11 REQUIRED)  # or `add_subdirectory(pybind11)`
+
+    add_executable(example main.cpp)
+    target_link_libraries(example PRIVATE pybind11::embed)
+
+The essential structure of the ``main.cpp`` file looks like this:
+
+.. code-block:: cpp
+
+    #include <pybind11/embed.h> // everything needed for embedding
+    namespace py = pybind11;
+
+    int main() {
+        py::scoped_interpreter guard{}; // start the interpreter and keep it alive
+
+        py::print("Hello, World!"); // use the Python API
+    }
+
+The interpreter must be initialized before using any Python API, which includes
+all the functions and classes in pybind11. The RAII guard class ``scoped_interpreter``
+takes care of the interpreter lifetime. After the guard is destroyed, the interpreter
+shuts down and clears its memory. No Python functions can be called after this.
+
+Executing Python code
+=====================
+
+There are a few different ways to run Python code. One option is to use ``eval``,
+``exec`` or ``eval_file``, as explained in :ref:`eval`. Here is a quick example in
+the context of an executable with an embedded interpreter:
+
+.. code-block:: cpp
+
+    #include <pybind11/embed.h>
+    namespace py = pybind11;
+
+    int main() {
+        py::scoped_interpreter guard{};
+
+        py::exec(R"(
+            kwargs = dict(name="World", number=42)
+            message = "Hello, {name}! The answer is {number}".format(**kwargs)
+            print(message)
+        )");
+    }
+
+Alternatively, similar results can be achieved using pybind11's API (see
+:doc:`/advanced/pycpp/index` for more details).
+
+.. code-block:: cpp
+
+    #include <pybind11/embed.h>
+    namespace py = pybind11;
+    using namespace py::literals;
+
+    int main() {
+        py::scoped_interpreter guard{};
+
+        auto kwargs = py::dict("name"_a="World", "number"_a=42);
+        auto message = "Hello, {name}! The answer is {number}"_s.format(**kwargs);
+        py::print(message);
+    }
+
+The two approaches can also be combined:
+
+.. code-block:: cpp
+
+    #include <pybind11/embed.h>
+    #include <iostream>
+
+    namespace py = pybind11;
+    using namespace py::literals;
+
+    int main() {
+        py::scoped_interpreter guard{};
+
+        auto locals = py::dict("name"_a="World", "number"_a=42);
+        py::exec(R"(
+            message = "Hello, {name}! The answer is {number}".format(**locals())
+        )", py::globals(), locals);
+
+        auto message = locals["message"].cast<std::string>();
+        std::cout << message;
+    }
+
+Importing modules
+=================
+
+Python modules can be imported using ``module_::import()``:
+
+.. code-block:: cpp
+
+    py::module_ sys = py::module_::import("sys");
+    py::print(sys.attr("path"));
+
+For convenience, the current working directory is included in ``sys.path`` when
+embedding the interpreter. This makes it easy to import local Python files:
+
+.. code-block:: python
+
+    """calc.py located in the working directory"""
+
+
+    def add(i, j):
+        return i + j
+
+
+.. code-block:: cpp
+
+    py::module_ calc = py::module_::import("calc");
+    py::object result = calc.attr("add")(1, 2);
+    int n = result.cast<int>();
+    assert(n == 3);
+
+Modules can be reloaded using ``module_::reload()`` if the source is modified e.g.
+by an external process. This can be useful in scenarios where the application
+imports a user defined data processing script which needs to be updated after
+changes by the user. Note that this function does not reload modules recursively.
+
+.. _embedding_modules:
+
+Adding embedded modules
+=======================
+
+Embedded binary modules can be added using the ``PYBIND11_EMBEDDED_MODULE`` macro.
+Note that the definition must be placed at global scope. They can be imported
+like any other module.
+
+.. code-block:: cpp
+
+    #include <pybind11/embed.h>
+    namespace py = pybind11;
+
+    PYBIND11_EMBEDDED_MODULE(fast_calc, m) {
+        // `m` is a `py::module_` which is used to bind functions and classes
+        m.def("add", [](int i, int j) {
+            return i + j;
+        });
+    }
+
+    int main() {
+        py::scoped_interpreter guard{};
+
+        auto fast_calc = py::module_::import("fast_calc");
+        auto result = fast_calc.attr("add")(1, 2).cast<int>();
+        assert(result == 3);
+    }
+
+Unlike extension modules where only a single binary module can be created, on
+the embedded side an unlimited number of modules can be added using multiple
+``PYBIND11_EMBEDDED_MODULE`` definitions (as long as they have unique names).
+
+These modules are added to Python's list of builtins, so they can also be
+imported in pure Python files loaded by the interpreter. Everything interacts
+naturally:
+
+.. code-block:: python
+
+    """py_module.py located in the working directory"""
+    import cpp_module
+
+    a = cpp_module.a
+    b = a + 1
+
+
+.. code-block:: cpp
+
+    #include <pybind11/embed.h>
+    namespace py = pybind11;
+
+    PYBIND11_EMBEDDED_MODULE(cpp_module, m) {
+        m.attr("a") = 1;
+    }
+
+    int main() {
+        py::scoped_interpreter guard{};
+
+        auto py_module = py::module_::import("py_module");
+
+        auto locals = py::dict("fmt"_a="{} + {} = {}", **py_module.attr("__dict__"));
+        assert(locals["a"].cast<int>() == 1);
+        assert(locals["b"].cast<int>() == 2);
+
+        py::exec(R"(
+            c = a + b
+            message = fmt.format(a, b, c)
+        )", py::globals(), locals);
+
+        assert(locals["c"].cast<int>() == 3);
+        assert(locals["message"].cast<std::string>() == "1 + 2 = 3");
+    }
+
+
+Interpreter lifetime
+====================
+
+The Python interpreter shuts down when ``scoped_interpreter`` is destroyed. After
+this, creating a new instance will restart the interpreter. Alternatively, the
+``initialize_interpreter`` / ``finalize_interpreter`` pair of functions can be used
+to directly set the state at any time.
+
+Modules created with pybind11 can be safely re-initialized after the interpreter
+has been restarted. However, this may not apply to third-party extension modules.
+The issue is that Python itself cannot completely unload extension modules and
+there are several caveats with regard to interpreter restarting. In short, not
+all memory may be freed, either due to Python reference cycles or user-created
+global data. All the details can be found in the CPython documentation.
+
+.. warning::
+
+    Creating two concurrent ``scoped_interpreter`` guards is a fatal error. So is
+    calling ``initialize_interpreter`` for a second time after the interpreter
+    has already been initialized.
+
+    Do not use the raw CPython API functions ``Py_Initialize`` and
+    ``Py_Finalize`` as these do not properly handle the lifetime of
+    pybind11's internal data.
+
+
+Sub-interpreter support
+=======================
+
+Creating multiple copies of ``scoped_interpreter`` is not possible because it
+represents the main Python interpreter. Sub-interpreters are something different
+and they do permit the existence of multiple interpreters. This is an advanced
+feature of the CPython API and should be handled with care. pybind11 does not
+currently offer a C++ interface for sub-interpreters, so refer to the CPython
+documentation for all the details regarding this feature.
+
+We'll just mention a couple of caveats the sub-interpreters support in pybind11:
+
+ 1. Sub-interpreters will not receive independent copies of embedded modules.
+    Instead, these are shared and modifications in one interpreter may be
+    reflected in another.
+
+ 2. Managing multiple threads, multiple interpreters and the GIL can be
+    challenging and there are several caveats here, even within the pure
+    CPython API (please refer to the Python docs for details). As for
+    pybind11, keep in mind that ``gil_scoped_release`` and ``gil_scoped_acquire``
+    do not take sub-interpreters into account.

libs/pybind/docs/advanced/exceptions.rst

@@ -0,0 +1,398 @@
+Exceptions
+##########
+
+Built-in C++ to Python exception translation
+============================================
+
+When Python calls C++ code through pybind11, pybind11 provides a C++ exception handler
+that will trap C++ exceptions, translate them to the corresponding Python exception,
+and raise them so that Python code can handle them.
+
+pybind11 defines translations for ``std::exception`` and its standard
+subclasses, and several special exception classes that translate to specific
+Python exceptions. Note that these are not actually Python exceptions, so they
+cannot be examined using the Python C API. Instead, they are pure C++ objects
+that pybind11 will translate the corresponding Python exception when they arrive
+at its exception handler.
+
+.. tabularcolumns:: |p{0.5\textwidth}|p{0.45\textwidth}|
+
++--------------------------------------+--------------------------------------+
+|  Exception thrown by C++             |  Translated to Python exception type |
++======================================+======================================+
+| :class:`std::exception`              | ``RuntimeError``                     |
++--------------------------------------+--------------------------------------+
+| :class:`std::bad_alloc`              | ``MemoryError``                      |
++--------------------------------------+--------------------------------------+
+| :class:`std::domain_error`           | ``ValueError``                       |
++--------------------------------------+--------------------------------------+
+| :class:`std::invalid_argument`       | ``ValueError``                       |
++--------------------------------------+--------------------------------------+
+| :class:`std::length_error`           | ``ValueError``                       |
++--------------------------------------+--------------------------------------+
+| :class:`std::out_of_range`           | ``IndexError``                       |
++--------------------------------------+--------------------------------------+
+| :class:`std::range_error`            | ``ValueError``                       |
++--------------------------------------+--------------------------------------+
+| :class:`std::overflow_error`         | ``OverflowError``                    |
++--------------------------------------+--------------------------------------+
+| :class:`pybind11::stop_iteration`    | ``StopIteration`` (used to implement |
+|                                      | custom iterators)                    |
++--------------------------------------+--------------------------------------+
+| :class:`pybind11::index_error`       | ``IndexError`` (used to indicate out |
+|                                      | of bounds access in ``__getitem__``, |
+|                                      | ``__setitem__``, etc.)               |
++--------------------------------------+--------------------------------------+
+| :class:`pybind11::key_error`         | ``KeyError`` (used to indicate out   |
+|                                      | of bounds access in ``__getitem__``, |
+|                                      | ``__setitem__`` in dict-like         |
+|                                      | objects, etc.)                       |
++--------------------------------------+--------------------------------------+
+| :class:`pybind11::value_error`       | ``ValueError`` (used to indicate     |
+|                                      | wrong value passed in                |
+|                                      | ``container.remove(...)``)           |
++--------------------------------------+--------------------------------------+
+| :class:`pybind11::type_error`        | ``TypeError``                        |
++--------------------------------------+--------------------------------------+
+| :class:`pybind11::buffer_error`      | ``BufferError``                      |
++--------------------------------------+--------------------------------------+
+| :class:`pybind11::import_error`      | ``ImportError``                      |
++--------------------------------------+--------------------------------------+
+| :class:`pybind11::attribute_error`   | ``AttributeError``                   |
++--------------------------------------+--------------------------------------+
+| Any other exception                  | ``RuntimeError``                     |
++--------------------------------------+--------------------------------------+
+
+Exception translation is not bidirectional. That is, *catching* the C++
+exceptions defined above will not trap exceptions that originate from
+Python. For that, catch :class:`pybind11::error_already_set`. See :ref:`below
+<handling_python_exceptions_cpp>` for further details.
+
+There is also a special exception :class:`cast_error` that is thrown by
+:func:`handle::call` when the input arguments cannot be converted to Python
+objects.
+
+Registering custom translators
+==============================
+
+If the default exception conversion policy described above is insufficient,
+pybind11 also provides support for registering custom exception translators.
+Similar to pybind11 classes, exception translators can be local to the module
+they are defined in or global to the entire python session.  To register a simple
+exception conversion that translates a C++ exception into a new Python exception
+using the C++ exception's ``what()`` method, a helper function is available:
+
+.. code-block:: cpp
+
+    py::register_exception<CppExp>(module, "PyExp");
+
+This call creates a Python exception class with the name ``PyExp`` in the given
+module and automatically converts any encountered exceptions of type ``CppExp``
+into Python exceptions of type ``PyExp``.
+
+A matching function is available for registering a local exception translator:
+
+.. code-block:: cpp
+
+    py::register_local_exception<CppExp>(module, "PyExp");
+
+
+It is possible to specify base class for the exception using the third
+parameter, a ``handle``:
+
+.. code-block:: cpp
+
+    py::register_exception<CppExp>(module, "PyExp", PyExc_RuntimeError);
+    py::register_local_exception<CppExp>(module, "PyExp", PyExc_RuntimeError);
+
+Then ``PyExp`` can be caught both as ``PyExp`` and ``RuntimeError``.
+
+The class objects of the built-in Python exceptions are listed in the Python
+documentation on `Standard Exceptions <https://docs.python.org/3/c-api/exceptions.html#standard-exceptions>`_.
+The default base class is ``PyExc_Exception``.
+
+When more advanced exception translation is needed, the functions
+``py::register_exception_translator(translator)`` and
+``py::register_local_exception_translator(translator)`` can be used to register
+functions that can translate arbitrary exception types (and which may include
+additional logic to do so).  The functions takes a stateless callable (e.g. a
+function pointer or a lambda function without captured variables) with the call
+signature ``void(std::exception_ptr)``.
+
+When a C++ exception is thrown, the registered exception translators are tried
+in reverse order of registration (i.e. the last registered translator gets the
+first shot at handling the exception). All local translators will be tried
+before a global translator is tried.
+
+Inside the translator, ``std::rethrow_exception`` should be used within
+a try block to re-throw the exception.  One or more catch clauses to catch
+the appropriate exceptions should then be used with each clause using
+``PyErr_SetString`` to set a Python exception or ``ex(string)`` to set
+the python exception to a custom exception type (see below).
+
+To declare a custom Python exception type, declare a ``py::exception`` variable
+and use this in the associated exception translator (note: it is often useful
+to make this a static declaration when using it inside a lambda expression
+without requiring capturing).
+
+The following example demonstrates this for a hypothetical exception classes
+``MyCustomException`` and ``OtherException``: the first is translated to a
+custom python exception ``MyCustomError``, while the second is translated to a
+standard python RuntimeError:
+
+.. code-block:: cpp
+
+    static py::exception<MyCustomException> exc(m, "MyCustomError");
+    py::register_exception_translator([](std::exception_ptr p) {
+        try {
+            if (p) std::rethrow_exception(p);
+        } catch (const MyCustomException &e) {
+            exc(e.what());
+        } catch (const OtherException &e) {
+            PyErr_SetString(PyExc_RuntimeError, e.what());
+        }
+    });
+
+Multiple exceptions can be handled by a single translator, as shown in the
+example above. If the exception is not caught by the current translator, the
+previously registered one gets a chance.
+
+If none of the registered exception translators is able to handle the
+exception, it is handled by the default converter as described in the previous
+section.
+
+.. seealso::
+
+    The file :file:`tests/test_exceptions.cpp` contains examples
+    of various custom exception translators and custom exception types.
+
+.. note::
+
+    Call either ``PyErr_SetString`` or a custom exception's call
+    operator (``exc(string)``) for every exception caught in a custom exception
+    translator.  Failure to do so will cause Python to crash with ``SystemError:
+    error return without exception set``.
+
+    Exceptions that you do not plan to handle should simply not be caught, or
+    may be explicitly (re-)thrown to delegate it to the other,
+    previously-declared existing exception translators.
+
+    Note that ``libc++`` and ``libstdc++`` `behave differently <https://stackoverflow.com/questions/19496643/using-clang-fvisibility-hidden-and-typeinfo-and-type-erasure/28827430>`_
+    with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI boundaries need to be explicitly exported, as exercised in ``tests/test_exceptions.h``.
+    See also: "Problems with C++ exceptions" under `GCC Wiki <https://gcc.gnu.org/wiki/Visibility>`_.
+
+
+Local vs Global Exception Translators
+=====================================
+
+When a global exception translator is registered, it will be applied across all
+modules in the reverse order of registration. This can create behavior where the
+order of module import influences how exceptions are translated.
+
+If module1 has the following translator:
+
+.. code-block:: cpp
+
+      py::register_exception_translator([](std::exception_ptr p) {
+        try {
+            if (p) std::rethrow_exception(p);
+        } catch (const std::invalid_argument &e) {
+            PyErr_SetString("module1 handled this")
+        }
+      }
+
+and module2 has the following similar translator:
+
+.. code-block:: cpp
+
+      py::register_exception_translator([](std::exception_ptr p) {
+        try {
+            if (p) std::rethrow_exception(p);
+        } catch (const std::invalid_argument &e) {
+            PyErr_SetString("module2 handled this")
+        }
+      }
+
+then which translator handles the invalid_argument will be determined by the
+order that module1 and module2 are imported. Since exception translators are
+applied in the reverse order of registration, which ever module was imported
+last will "win" and that translator will be applied.
+
+If there are multiple pybind11 modules that share exception types (either
+standard built-in or custom) loaded into a single python instance and
+consistent error handling behavior is needed, then local translators should be
+used.
+
+Changing the previous example to use ``register_local_exception_translator``
+would mean that when invalid_argument is thrown in the module2 code, the
+module2 translator will always handle it, while in module1, the module1
+translator will do the same.
+
+.. _handling_python_exceptions_cpp:
+
+Handling exceptions from Python in C++
+======================================
+
+When C++ calls Python functions, such as in a callback function or when
+manipulating Python objects, and Python raises an ``Exception``, pybind11
+converts the Python exception into a C++ exception of type
+:class:`pybind11::error_already_set` whose payload contains a C++ string textual
+summary and the actual Python exception. ``error_already_set`` is used to
+propagate Python exception back to Python (or possibly, handle them in C++).
+
+.. tabularcolumns:: |p{0.5\textwidth}|p{0.45\textwidth}|
+
++--------------------------------------+--------------------------------------+
+|  Exception raised in Python          |  Thrown as C++ exception type        |
++======================================+======================================+
+| Any Python ``Exception``             | :class:`pybind11::error_already_set` |
++--------------------------------------+--------------------------------------+
+
+For example:
+
+.. code-block:: cpp
+
+    try {
+        // open("missing.txt", "r")
+        auto file = py::module_::import("io").attr("open")("missing.txt", "r");
+        auto text = file.attr("read")();
+        file.attr("close")();
+    } catch (py::error_already_set &e) {
+        if (e.matches(PyExc_FileNotFoundError)) {
+            py::print("missing.txt not found");
+        } else if (e.matches(PyExc_PermissionError)) {
+            py::print("missing.txt found but not accessible");
+        } else {
+            throw;
+        }
+    }
+
+Note that C++ to Python exception translation does not apply here, since that is
+a method for translating C++ exceptions to Python, not vice versa. The error raised
+from Python is always ``error_already_set``.
+
+This example illustrates this behavior:
+
+.. code-block:: cpp
+
+    try {
+        py::eval("raise ValueError('The Ring')");
+    } catch (py::value_error &boromir) {
+        // Boromir never gets the ring
+        assert(false);
+    } catch (py::error_already_set &frodo) {
+        // Frodo gets the ring
+        py::print("I will take the ring");
+    }
+
+    try {
+        // py::value_error is a request for pybind11 to raise a Python exception
+        throw py::value_error("The ball");
+    } catch (py::error_already_set &cat) {
+        // cat won't catch the ball since
+        // py::value_error is not a Python exception
+        assert(false);
+    } catch (py::value_error &dog) {
+        // dog will catch the ball
+        py::print("Run Spot run");
+        throw;  // Throw it again (pybind11 will raise ValueError)
+    }
+
+Handling errors from the Python C API
+=====================================
+
+Where possible, use :ref:`pybind11 wrappers <wrappers>` instead of calling
+the Python C API directly. When calling the Python C API directly, in
+addition to manually managing reference counts, one must follow the pybind11
+error protocol, which is outlined here.
+
+After calling the Python C API, if Python returns an error,
+``throw py::error_already_set();``, which allows pybind11 to deal with the
+exception and pass it back to the Python interpreter. This includes calls to
+the error setting functions such as ``PyErr_SetString``.
+
+.. code-block:: cpp
+
+    PyErr_SetString(PyExc_TypeError, "C API type error demo");
+    throw py::error_already_set();
+
+    // But it would be easier to simply...
+    throw py::type_error("pybind11 wrapper type error");
+
+Alternately, to ignore the error, call `PyErr_Clear
+<https://docs.python.org/3/c-api/exceptions.html#c.PyErr_Clear>`_.
+
+Any Python error must be thrown or cleared, or Python/pybind11 will be left in
+an invalid state.
+
+Chaining exceptions ('raise from')
+==================================
+
+Python has a mechanism for indicating that exceptions were caused by other
+exceptions:
+
+.. code-block:: py
+
+    try:
+        print(1 / 0)
+    except Exception as exc:
+        raise RuntimeError("could not divide by zero") from exc
+
+To do a similar thing in pybind11, you can use the ``py::raise_from`` function. It
+sets the current python error indicator, so to continue propagating the exception
+you should ``throw py::error_already_set()``.
+
+.. code-block:: cpp
+
+    try {
+        py::eval("print(1 / 0"));
+    } catch (py::error_already_set &e) {
+        py::raise_from(e, PyExc_RuntimeError, "could not divide by zero");
+        throw py::error_already_set();
+    }
+
+.. versionadded:: 2.8
+
+.. _unraisable_exceptions:
+
+Handling unraisable exceptions
+==============================
+
+If a Python function invoked from a C++ destructor or any function marked
+``noexcept(true)`` (collectively, "noexcept functions") throws an exception, there
+is no way to propagate the exception, as such functions may not throw.
+Should they throw or fail to catch any exceptions in their call graph,
+the C++ runtime calls ``std::terminate()`` to abort immediately.
+
+Similarly, Python exceptions raised in a class's ``__del__`` method do not
+propagate, but are logged by Python as an unraisable error. In Python 3.8+, a
+`system hook is triggered
+<https://docs.python.org/3/library/sys.html#sys.unraisablehook>`_
+and an auditing event is logged.
+
+Any noexcept function should have a try-catch block that traps
+class:`error_already_set` (or any other exception that can occur). Note that
+pybind11 wrappers around Python exceptions such as
+:class:`pybind11::value_error` are *not* Python exceptions; they are C++
+exceptions that pybind11 catches and converts to Python exceptions. Noexcept
+functions cannot propagate these exceptions either. A useful approach is to
+convert them to Python exceptions and then ``discard_as_unraisable`` as shown
+below.
+
+.. code-block:: cpp
+
+    void nonthrowing_func() noexcept(true) {
+        try {
+            // ...
+        } catch (py::error_already_set &eas) {
+            // Discard the Python error using Python APIs, using the C++ magic
+            // variable __func__. Python already knows the type and value and of the
+            // exception object.
+            eas.discard_as_unraisable(__func__);
+        } catch (const std::exception &e) {
+            // Log and discard C++ exceptions.
+            third_party::log(e);
+        }
+    }
+
+.. versionadded:: 2.6

libs/pybind/docs/advanced/functions.rst

@@ -0,0 +1,614 @@
+Functions
+#########
+
+Before proceeding with this section, make sure that you are already familiar
+with the basics of binding functions and classes, as explained in :doc:`/basics`
+and :doc:`/classes`. The following guide is applicable to both free and member
+functions, i.e. *methods* in Python.
+
+.. _return_value_policies:
+
+Return value policies
+=====================
+
+Python and C++ use fundamentally different ways of managing the memory and
+lifetime of objects managed by them. This can lead to issues when creating
+bindings for functions that return a non-trivial type. Just by looking at the
+type information, it is not clear whether Python should take charge of the
+returned value and eventually free its resources, or if this is handled on the
+C++ side. For this reason, pybind11 provides a several *return value policy*
+annotations that can be passed to the :func:`module_::def` and
+:func:`class_::def` functions. The default policy is
+:enum:`return_value_policy::automatic`.
+
+Return value policies are tricky, and it's very important to get them right.
+Just to illustrate what can go wrong, consider the following simple example:
+
+.. code-block:: cpp
+
+    /* Function declaration */
+    Data *get_data() { return _data; /* (pointer to a static data structure) */ }
+    ...
+
+    /* Binding code */
+    m.def("get_data", &get_data); // <-- KABOOM, will cause crash when called from Python
+
+What's going on here? When ``get_data()`` is called from Python, the return
+value (a native C++ type) must be wrapped to turn it into a usable Python type.
+In this case, the default return value policy (:enum:`return_value_policy::automatic`)
+causes pybind11 to assume ownership of the static ``_data`` instance.
+
+When Python's garbage collector eventually deletes the Python
+wrapper, pybind11 will also attempt to delete the C++ instance (via ``operator
+delete()``) due to the implied ownership. At this point, the entire application
+will come crashing down, though errors could also be more subtle and involve
+silent data corruption.
+
+In the above example, the policy :enum:`return_value_policy::reference` should have
+been specified so that the global data instance is only *referenced* without any
+implied transfer of ownership, i.e.:
+
+.. code-block:: cpp
+
+    m.def("get_data", &get_data, py::return_value_policy::reference);
+
+On the other hand, this is not the right policy for many other situations,
+where ignoring ownership could lead to resource leaks.
+As a developer using pybind11, it's important to be familiar with the different
+return value policies, including which situation calls for which one of them.
+The following table provides an overview of available policies:
+
+.. tabularcolumns:: |p{0.5\textwidth}|p{0.45\textwidth}|
+
++--------------------------------------------------+----------------------------------------------------------------------------+
+| Return value policy                              | Description                                                                |
++==================================================+============================================================================+
+| :enum:`return_value_policy::take_ownership`      | Reference an existing object (i.e. do not create a new copy) and take      |
+|                                                  | ownership. Python will call the destructor and delete operator when the    |
+|                                                  | object's reference count reaches zero. Undefined behavior ensues when the  |
+|                                                  | C++ side does the same, or when the data was not dynamically allocated.    |
++--------------------------------------------------+----------------------------------------------------------------------------+
+| :enum:`return_value_policy::copy`                | Create a new copy of the returned object, which will be owned by Python.   |
+|                                                  | This policy is comparably safe because the lifetimes of the two instances  |
+|                                                  | are decoupled.                                                             |
++--------------------------------------------------+----------------------------------------------------------------------------+
+| :enum:`return_value_policy::move`                | Use ``std::move`` to move the return value contents into a new instance    |
+|                                                  | that will be owned by Python. This policy is comparably safe because the   |
+|                                                  | lifetimes of the two instances (move source and destination) are decoupled.|
++--------------------------------------------------+----------------------------------------------------------------------------+
+| :enum:`return_value_policy::reference`           | Reference an existing object, but do not take ownership. The C++ side is   |
+|                                                  | responsible for managing the object's lifetime and deallocating it when    |
+|                                                  | it is no longer used. Warning: undefined behavior will ensue when the C++  |
+|                                                  | side deletes an object that is still referenced and used by Python.        |
++--------------------------------------------------+----------------------------------------------------------------------------+
+| :enum:`return_value_policy::reference_internal`  | Indicates that the lifetime of the return value is tied to the lifetime    |
+|                                                  | of a parent object, namely the implicit ``this``, or ``self`` argument of  |
+|                                                  | the called method or property. Internally, this policy works just like     |
+|                                                  | :enum:`return_value_policy::reference` but additionally applies a          |
+|                                                  | ``keep_alive<0, 1>`` *call policy* (described in the next section) that    |
+|                                                  | prevents the parent object from being garbage collected as long as the     |
+|                                                  | return value is referenced by Python. This is the default policy for       |
+|                                                  | property getters created via ``def_property``, ``def_readwrite``, etc.     |
++--------------------------------------------------+----------------------------------------------------------------------------+
+| :enum:`return_value_policy::automatic`           | This policy falls back to the policy                                       |
+|                                                  | :enum:`return_value_policy::take_ownership` when the return value is a     |
+|                                                  | pointer. Otherwise, it uses :enum:`return_value_policy::move` or           |
+|                                                  | :enum:`return_value_policy::copy` for rvalue and lvalue references,        |
+|                                                  | respectively. See above for a description of what all of these different   |
+|                                                  | policies do. This is the default policy for ``py::class_``-wrapped types.  |
++--------------------------------------------------+----------------------------------------------------------------------------+
+| :enum:`return_value_policy::automatic_reference` | As above, but use policy :enum:`return_value_policy::reference` when the   |
+|                                                  | return value is a pointer. This is the default conversion policy for       |
+|                                                  | function arguments when calling Python functions manually from C++ code    |
+|                                                  | (i.e. via ``handle::operator()``) and the casters in ``pybind11/stl.h``.   |
+|                                                  | You probably won't need to use this explicitly.                            |
++--------------------------------------------------+----------------------------------------------------------------------------+
+
+Return value policies can also be applied to properties:
+
+.. code-block:: cpp
+
+    class_<MyClass>(m, "MyClass")
+        .def_property("data", &MyClass::getData, &MyClass::setData,
+                      py::return_value_policy::copy);
+
+Technically, the code above applies the policy to both the getter and the
+setter function, however, the setter doesn't really care about *return*
+value policies which makes this a convenient terse syntax. Alternatively,
+targeted arguments can be passed through the :class:`cpp_function` constructor:
+
+.. code-block:: cpp
+
+    class_<MyClass>(m, "MyClass")
+        .def_property("data",
+            py::cpp_function(&MyClass::getData, py::return_value_policy::copy),
+            py::cpp_function(&MyClass::setData)
+        );
+
+.. warning::
+
+    Code with invalid return value policies might access uninitialized memory or
+    free data structures multiple times, which can lead to hard-to-debug
+    non-determinism and segmentation faults, hence it is worth spending the
+    time to understand all the different options in the table above.
+
+.. note::
+
+    One important aspect of the above policies is that they only apply to
+    instances which pybind11 has *not* seen before, in which case the policy
+    clarifies essential questions about the return value's lifetime and
+    ownership.  When pybind11 knows the instance already (as identified by its
+    type and address in memory), it will return the existing Python object
+    wrapper rather than creating a new copy.
+
+.. note::
+
+    The next section on :ref:`call_policies` discusses *call policies* that can be
+    specified *in addition* to a return value policy from the list above. Call
+    policies indicate reference relationships that can involve both return values
+    and parameters of functions.
+
+.. note::
+
+   As an alternative to elaborate call policies and lifetime management logic,
+   consider using smart pointers (see the section on :ref:`smart_pointers` for
+   details). Smart pointers can tell whether an object is still referenced from
+   C++ or Python, which generally eliminates the kinds of inconsistencies that
+   can lead to crashes or undefined behavior. For functions returning smart
+   pointers, it is not necessary to specify a return value policy.
+
+.. _call_policies:
+
+Additional call policies
+========================
+
+In addition to the above return value policies, further *call policies* can be
+specified to indicate dependencies between parameters or ensure a certain state
+for the function call.
+
+Keep alive
+----------
+
+In general, this policy is required when the C++ object is any kind of container
+and another object is being added to the container. ``keep_alive<Nurse, Patient>``
+indicates that the argument with index ``Patient`` should be kept alive at least
+until the argument with index ``Nurse`` is freed by the garbage collector. Argument
+indices start at one, while zero refers to the return value. For methods, index
+``1`` refers to the implicit ``this`` pointer, while regular arguments begin at
+index ``2``. Arbitrarily many call policies can be specified. When a ``Nurse``
+with value ``None`` is detected at runtime, the call policy does nothing.
+
+When the nurse is not a pybind11-registered type, the implementation internally
+relies on the ability to create a *weak reference* to the nurse object. When
+the nurse object is not a pybind11-registered type and does not support weak
+references, an exception will be thrown.
+
+If you use an incorrect argument index, you will get a ``RuntimeError`` saying
+``Could not activate keep_alive!``. You should review the indices you're using.
+
+Consider the following example: here, the binding code for a list append
+operation ties the lifetime of the newly added element to the underlying
+container:
+
+.. code-block:: cpp
+
+    py::class_<List>(m, "List")
+        .def("append", &List::append, py::keep_alive<1, 2>());
+
+For consistency, the argument indexing is identical for constructors. Index
+``1`` still refers to the implicit ``this`` pointer, i.e. the object which is
+being constructed. Index ``0`` refers to the return type which is presumed to
+be ``void`` when a constructor is viewed like a function. The following example
+ties the lifetime of the constructor element to the constructed object:
+
+.. code-block:: cpp
+
+    py::class_<Nurse>(m, "Nurse")
+        .def(py::init<Patient &>(), py::keep_alive<1, 2>());
+
+.. note::
+
+    ``keep_alive`` is analogous to the ``with_custodian_and_ward`` (if Nurse,
+    Patient != 0) and ``with_custodian_and_ward_postcall`` (if Nurse/Patient ==
+    0) policies from Boost.Python.
+
+Call guard
+----------
+
+The ``call_guard<T>`` policy allows any scope guard type ``T`` to be placed
+around the function call. For example, this definition:
+
+.. code-block:: cpp
+
+    m.def("foo", foo, py::call_guard<T>());
+
+is equivalent to the following pseudocode:
+
+.. code-block:: cpp
+
+    m.def("foo", [](args...) {
+        T scope_guard;
+        return foo(args...); // forwarded arguments
+    });
+
+The only requirement is that ``T`` is default-constructible, but otherwise any
+scope guard will work. This is very useful in combination with ``gil_scoped_release``.
+See :ref:`gil`.
+
+Multiple guards can also be specified as ``py::call_guard<T1, T2, T3...>``. The
+constructor order is left to right and destruction happens in reverse.
+
+.. seealso::
+
+    The file :file:`tests/test_call_policies.cpp` contains a complete example
+    that demonstrates using `keep_alive` and `call_guard` in more detail.
+
+.. _python_objects_as_args:
+
+Python objects as arguments
+===========================
+
+pybind11 exposes all major Python types using thin C++ wrapper classes. These
+wrapper classes can also be used as parameters of functions in bindings, which
+makes it possible to directly work with native Python types on the C++ side.
+For instance, the following statement iterates over a Python ``dict``:
+
+.. code-block:: cpp
+
+    void print_dict(const py::dict& dict) {
+        /* Easily interact with Python types */
+        for (auto item : dict)
+            std::cout << "key=" << std::string(py::str(item.first)) << ", "
+                      << "value=" << std::string(py::str(item.second)) << std::endl;
+    }
+
+It can be exported:
+
+.. code-block:: cpp
+
+    m.def("print_dict", &print_dict);
+
+And used in Python as usual:
+
+.. code-block:: pycon
+
+    >>> print_dict({"foo": 123, "bar": "hello"})
+    key=foo, value=123
+    key=bar, value=hello
+
+For more information on using Python objects in C++, see :doc:`/advanced/pycpp/index`.
+
+Accepting \*args and \*\*kwargs
+===============================
+
+Python provides a useful mechanism to define functions that accept arbitrary
+numbers of arguments and keyword arguments:
+
+.. code-block:: python
+
+   def generic(*args, **kwargs):
+       ...  # do something with args and kwargs
+
+Such functions can also be created using pybind11:
+
+.. code-block:: cpp
+
+   void generic(py::args args, const py::kwargs& kwargs) {
+       /// .. do something with args
+       if (kwargs)
+           /// .. do something with kwargs
+   }
+
+   /// Binding code
+   m.def("generic", &generic);
+
+The class ``py::args`` derives from ``py::tuple`` and ``py::kwargs`` derives
+from ``py::dict``.
+
+You may also use just one or the other, and may combine these with other
+arguments.  Note, however, that ``py::kwargs`` must always be the last argument
+of the function, and ``py::args`` implies that any further arguments are
+keyword-only (see :ref:`keyword_only_arguments`).
+
+Please refer to the other examples for details on how to iterate over these,
+and on how to cast their entries into C++ objects. A demonstration is also
+available in ``tests/test_kwargs_and_defaults.cpp``.
+
+.. note::
+
+    When combining \*args or \*\*kwargs with :ref:`keyword_args` you should
+    *not* include ``py::arg`` tags for the ``py::args`` and ``py::kwargs``
+    arguments.
+
+Default arguments revisited
+===========================
+
+The section on :ref:`default_args` previously discussed basic usage of default
+arguments using pybind11. One noteworthy aspect of their implementation is that
+default arguments are converted to Python objects right at declaration time.
+Consider the following example:
+
+.. code-block:: cpp
+
+    py::class_<MyClass>("MyClass")
+        .def("myFunction", py::arg("arg") = SomeType(123));
+
+In this case, pybind11 must already be set up to deal with values of the type
+``SomeType`` (via a prior instantiation of ``py::class_<SomeType>``), or an
+exception will be thrown.
+
+Another aspect worth highlighting is that the "preview" of the default argument
+in the function signature is generated using the object's ``__repr__`` method.
+If not available, the signature may not be very helpful, e.g.:
+
+.. code-block:: pycon
+
+    FUNCTIONS
+    ...
+    |  myFunction(...)
+    |      Signature : (MyClass, arg : SomeType = <SomeType object at 0x101b7b080>) -> NoneType
+    ...
+
+The first way of addressing this is by defining ``SomeType.__repr__``.
+Alternatively, it is possible to specify the human-readable preview of the
+default argument manually using the ``arg_v`` notation:
+
+.. code-block:: cpp
+
+    py::class_<MyClass>("MyClass")
+        .def("myFunction", py::arg_v("arg", SomeType(123), "SomeType(123)"));
+
+Sometimes it may be necessary to pass a null pointer value as a default
+argument. In this case, remember to cast it to the underlying type in question,
+like so:
+
+.. code-block:: cpp
+
+    py::class_<MyClass>("MyClass")
+        .def("myFunction", py::arg("arg") = static_cast<SomeType *>(nullptr));
+
+.. _keyword_only_arguments:
+
+Keyword-only arguments
+======================
+
+Python implements keyword-only arguments by specifying an unnamed ``*``
+argument in a function definition:
+
+.. code-block:: python
+
+    def f(a, *, b):  # a can be positional or via keyword; b must be via keyword
+        pass
+
+
+    f(a=1, b=2)  # good
+    f(b=2, a=1)  # good
+    f(1, b=2)  # good
+    f(1, 2)  # TypeError: f() takes 1 positional argument but 2 were given
+
+Pybind11 provides a ``py::kw_only`` object that allows you to implement
+the same behaviour by specifying the object between positional and keyword-only
+argument annotations when registering the function:
+
+.. code-block:: cpp
+
+    m.def("f", [](int a, int b) { /* ... */ },
+          py::arg("a"), py::kw_only(), py::arg("b"));
+
+.. versionadded:: 2.6
+
+A ``py::args`` argument implies that any following arguments are keyword-only,
+as if ``py::kw_only()`` had been specified in the same relative location of the
+argument list as the ``py::args`` argument.  The ``py::kw_only()`` may be
+included to be explicit about this, but is not required.
+
+.. versionchanged:: 2.9
+   This can now be combined with ``py::args``. Before, ``py::args`` could only
+   occur at the end of the argument list, or immediately before a ``py::kwargs``
+   argument at the end.
+
+
+Positional-only arguments
+=========================
+
+Python 3.8 introduced a new positional-only argument syntax, using ``/`` in the
+function definition (note that this has been a convention for CPython
+positional arguments, such as in ``pow()``, since Python 2). You can
+do the same thing in any version of Python using ``py::pos_only()``:
+
+.. code-block:: cpp
+
+   m.def("f", [](int a, int b) { /* ... */ },
+          py::arg("a"), py::pos_only(), py::arg("b"));
+
+You now cannot give argument ``a`` by keyword. This can be combined with
+keyword-only arguments, as well.
+
+.. versionadded:: 2.6
+
+.. _nonconverting_arguments:
+
+Non-converting arguments
+========================
+
+Certain argument types may support conversion from one type to another.  Some
+examples of conversions are:
+
+* :ref:`implicit_conversions` declared using ``py::implicitly_convertible<A,B>()``
+* Calling a method accepting a double with an integer argument
+* Calling a ``std::complex<float>`` argument with a non-complex python type
+  (for example, with a float).  (Requires the optional ``pybind11/complex.h``
+  header).
+* Calling a function taking an Eigen matrix reference with a numpy array of the
+  wrong type or of an incompatible data layout.  (Requires the optional
+  ``pybind11/eigen.h`` header).
+
+This behaviour is sometimes undesirable: the binding code may prefer to raise
+an error rather than convert the argument.  This behaviour can be obtained
+through ``py::arg`` by calling the ``.noconvert()`` method of the ``py::arg``
+object, such as:
+
+.. code-block:: cpp
+
+    m.def("floats_only", [](double f) { return 0.5 * f; }, py::arg("f").noconvert());
+    m.def("floats_preferred", [](double f) { return 0.5 * f; }, py::arg("f"));
+
+Attempting the call the second function (the one without ``.noconvert()``) with
+an integer will succeed, but attempting to call the ``.noconvert()`` version
+will fail with a ``TypeError``:
+
+.. code-block:: pycon
+
+    >>> floats_preferred(4)
+    2.0
+    >>> floats_only(4)
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+    TypeError: floats_only(): incompatible function arguments. The following argument types are supported:
+        1. (f: float) -> float
+
+    Invoked with: 4
+
+You may, of course, combine this with the :var:`_a` shorthand notation (see
+:ref:`keyword_args`) and/or :ref:`default_args`.  It is also permitted to omit
+the argument name by using the ``py::arg()`` constructor without an argument
+name, i.e. by specifying ``py::arg().noconvert()``.
+
+.. note::
+
+    When specifying ``py::arg`` options it is necessary to provide the same
+    number of options as the bound function has arguments.  Thus if you want to
+    enable no-convert behaviour for just one of several arguments, you will
+    need to specify a ``py::arg()`` annotation for each argument with the
+    no-convert argument modified to ``py::arg().noconvert()``.
+
+.. _none_arguments:
+
+Allow/Prohibiting None arguments
+================================
+
+When a C++ type registered with :class:`py::class_` is passed as an argument to
+a function taking the instance as pointer or shared holder (e.g. ``shared_ptr``
+or a custom, copyable holder as described in :ref:`smart_pointers`), pybind
+allows ``None`` to be passed from Python which results in calling the C++
+function with ``nullptr`` (or an empty holder) for the argument.
+
+To explicitly enable or disable this behaviour, using the
+``.none`` method of the :class:`py::arg` object:
+
+.. code-block:: cpp
+
+    py::class_<Dog>(m, "Dog").def(py::init<>());
+    py::class_<Cat>(m, "Cat").def(py::init<>());
+    m.def("bark", [](Dog *dog) -> std::string {
+        if (dog) return "woof!"; /* Called with a Dog instance */
+        else return "(no dog)"; /* Called with None, dog == nullptr */
+    }, py::arg("dog").none(true));
+    m.def("meow", [](Cat *cat) -> std::string {
+        // Can't be called with None argument
+        return "meow";
+    }, py::arg("cat").none(false));
+
+With the above, the Python call ``bark(None)`` will return the string ``"(no
+dog)"``, while attempting to call ``meow(None)`` will raise a ``TypeError``:
+
+.. code-block:: pycon
+
+    >>> from animals import Dog, Cat, bark, meow
+    >>> bark(Dog())
+    'woof!'
+    >>> meow(Cat())
+    'meow'
+    >>> bark(None)
+    '(no dog)'
+    >>> meow(None)
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+    TypeError: meow(): incompatible function arguments. The following argument types are supported:
+        1. (cat: animals.Cat) -> str
+
+    Invoked with: None
+
+The default behaviour when the tag is unspecified is to allow ``None``.
+
+.. note::
+
+    Even when ``.none(true)`` is specified for an argument, ``None`` will be converted to a
+    ``nullptr`` *only* for custom and :ref:`opaque <opaque>` types. Pointers to built-in types
+    (``double *``, ``int *``, ...) and STL types (``std::vector<T> *``, ...; if ``pybind11/stl.h``
+    is included) are copied when converted to C++ (see :doc:`/advanced/cast/overview`) and will
+    not allow ``None`` as argument.  To pass optional argument of these copied types consider
+    using ``std::optional<T>``
+
+.. _overload_resolution:
+
+Overload resolution order
+=========================
+
+When a function or method with multiple overloads is called from Python,
+pybind11 determines which overload to call in two passes.  The first pass
+attempts to call each overload without allowing argument conversion (as if
+every argument had been specified as ``py::arg().noconvert()`` as described
+above).
+
+If no overload succeeds in the no-conversion first pass, a second pass is
+attempted in which argument conversion is allowed (except where prohibited via
+an explicit ``py::arg().noconvert()`` attribute in the function definition).
+
+If the second pass also fails a ``TypeError`` is raised.
+
+Within each pass, overloads are tried in the order they were registered with
+pybind11. If the ``py::prepend()`` tag is added to the definition, a function
+can be placed at the beginning of the overload sequence instead, allowing user
+overloads to proceed built in functions.
+
+What this means in practice is that pybind11 will prefer any overload that does
+not require conversion of arguments to an overload that does, but otherwise
+prefers earlier-defined overloads to later-defined ones.
+
+.. note::
+
+    pybind11 does *not* further prioritize based on the number/pattern of
+    overloaded arguments.  That is, pybind11 does not prioritize a function
+    requiring one conversion over one requiring three, but only prioritizes
+    overloads requiring no conversion at all to overloads that require
+    conversion of at least one argument.
+
+.. versionadded:: 2.6
+
+    The ``py::prepend()`` tag.
+
+Binding functions with template parameters
+==========================================
+
+You can bind functions that have template parameters. Here's a function:
+
+.. code-block:: cpp
+
+    template <typename T>
+    void set(T t);
+
+C++ templates cannot be instantiated at runtime, so you cannot bind the
+non-instantiated function:
+
+.. code-block:: cpp
+
+    // BROKEN (this will not compile)
+    m.def("set", &set);
+
+You must bind each instantiated function template separately. You may bind
+each instantiation with the same name, which will be treated the same as
+an overloaded function:
+
+.. code-block:: cpp
+
+    m.def("set", &set<int>);
+    m.def("set", &set<std::string>);
+
+Sometimes it's more clear to bind them with separate names, which is also
+an option:
+
+.. code-block:: cpp
+
+    m.def("setInt", &set<int>);
+    m.def("setString", &set<std::string>);