Nanosecond times in Python (#522)

* initital implementation

* datetime replaces with sls::Duration in Python C bindings

* using custom type caster

* fix for conversion to seconds

* added set_count in python

* common header for pybind11 includes

authored-by: Erik Frojdh <erik.frojdh@psi.ch>

RELEASE.txt

@@ -96,7 +96,7 @@ This document describes the differences between v7.0.0 and v6.x.x
 
 2. Resolved Issues
 ==================
-
+    - Reading back sub-microsecond exposure times from the Python API. 
 
 3. Firmware Requirements
 ========================

python/CMakeLists.txt

@@ -9,6 +9,8 @@ pybind11_add_module(_slsdet
     src/pattern.cpp
     src/scan.cpp
     src/current.cpp
+    src/duration.cpp
+    src/DurationWrapper.cpp
 )
 
 target_link_libraries(_slsdet PUBLIC 

python/scripts/basic.py

@@ -3,22 +3,20 @@
 import os
 import sys
 import numpy as np
-from pathlib import Path
-sys.path.append(os.path.join(os.getcwd(), 'bin'))
+# from pathlib import Path
+# sys.path.append(os.path.join(os.getcwd(), 'bin'))
 
 from slsdet import Detector, Mythen3, Eiger, Jungfrau, DetectorDacs, Dac, Ctb, Gotthard2, Moench
 from slsdet import dacIndex, readoutMode
 from slsdet.lookup import view, find
 import slsdet
 
-
+from slsdet import DurationWrapper
 
 
 d = Detector()
-e = Eiger()
-c = Ctb()
-g = Gotthard2()
-# j = Jungfrau()
-# m = Mythen3()
-m = Moench()
+t = DurationWrapper(1.0)
 
+d.setExptime(t)
+
+# d.setExptime(0.3)

python/scripts/generate_functions.py

@@ -16,33 +16,40 @@ import time
 from pathlib import Path
 from parse import system_include_paths, clang_format_version
 
-REDC = '\033[91m'
-GREENC = '\033[92m'
-ENDC = '\033[0m'
+REDC = "\033[91m"
+GREENC = "\033[92m"
+ENDC = "\033[0m"
+
+
 def red(msg):
-    return f'{REDC}{msg}{ENDC}'
+    return f"{REDC}{msg}{ENDC}"
+
 
 def green(msg):
-    return f'{GREENC}{msg}{ENDC}'
+    return f"{GREENC}{msg}{ENDC}"
+
 
 def check_clang_format_version(required_version):
     if (ver := clang_format_version()) != required_version:
-        msg = red(f'Clang format version {required_version} required, detected: {ver}. Bye!')
+        msg = red(
+            f"Clang format version {required_version} required, detected: {ver}. Bye!"
+        )
         print(msg)
         sys.exit(1)
     else:
-        msg = green(f'Found clang-format version {ver}')
+        msg = green(f"Found clang-format version {ver}")
         print(msg)
 
+
 def check_for_compile_commands_json(path):
     # print(f"Looking for compile data base in: {path}")
-    compile_data_base_file = path/'compile_commands.json'
+    compile_data_base_file = path / "compile_commands.json"
     if not compile_data_base_file.exists():
         msg = red(f"No compile_commands.json file found in {path}. Bye!")
         print(msg)
         sys.exit(1)
     else:
-        msg = green(f'Found: {compile_data_base_file}')
+        msg = green(f"Found: {compile_data_base_file}")
         print(msg)
 
 
@@ -50,15 +57,13 @@ default_build_path = "/home/l_frojdh/sls/build/"
 fpath = "../../slsDetectorSoftware/src/Detector.cpp"
 
 
-
-
-
 m = []
 ag = []
 lines = []
 ag2 = []
 cn = []
 
+
 def get_arguments(node):
     args = [a.type.spelling for a in node.get_arguments()]
     args = [
@@ -70,18 +75,19 @@ def get_arguments(node):
         args = f", {args}"
     return args
 
+
 def get_arguments_with_default(node):
     args = []
     for arg in node.get_arguments():
         tokens = [t.spelling for t in arg.get_tokens()]
         # print(tokens)
-        if '=' in tokens:
-            if arg.type.spelling == "sls::Positions": #TODO! automate
+        if "=" in tokens:
+            if arg.type.spelling == "sls::Positions":  # TODO! automate
                 args.append("py::arg() = Positions{}")
             else:
-                args.append('py::arg()' + ''.join(tokens[tokens.index('='):]))
+                args.append("py::arg()" + "".join(tokens[tokens.index("=") :]))
         else:
-            args.append('py::arg()')
+            args.append("py::arg()")
     args = ", ".join(args)
     if args:
         args = f", {args}"
@@ -93,17 +99,26 @@ def get_fdec(node):
     if node.result_type.spelling:
         return_type = node.result_type.spelling
     else:
-        return_type = 'void'
-    
+        return_type = "void"
+
     if node.is_const_method():
-        const = 'const'
+        const = "const"
     else:
-        const = ''
+        const = ""
     args = ", ".join(args)
-    args = f'({return_type}(Detector::*)({args}){const})'
+    args = f"({return_type}(Detector::*)({args}){const})"
     return args
 
 
+def time_return_lambda(node, args):
+    names = ['a', 'b', 'c', 'd']
+    fa = [a.type.spelling for a in node.get_arguments()]
+    ca = ','.join(f'{arg} {n}' for arg, n in zip(fa, names))
+    na = ','.join(names[0:len(fa)])
+    s = f'CppDetectorApi.def("{node.spelling}",[](sls::Detector& self, {ca}){{ auto r = self.{node.spelling}({na}); \n return std::vector<sls::Duration>(r.begin(), r.end()); }}{args});'
+    return s
+
+
 def visit(node):
     if node.kind == cindex.CursorKind.CLASS_DECL:
         if node.displayname == "Detector":
@@ -113,15 +128,15 @@ def visit(node):
                     and child.access_specifier == cindex.AccessSpecifier.PUBLIC
                 ):
                     m.append(child)
-                    # args = get_arguments(child)
                     args = get_arguments_with_default(child)
                     fs = get_fdec(child)
                     lines.append(
-                        f'.def("{child.spelling}",{fs} &Detector::{child.spelling}{args})'
+                        f'CppDetectorApi.def("{child.spelling}",{fs} &Detector::{child.spelling}{args});'
                     )
                     if cargs.verbose:
-                        print(f'&Detector::{child.spelling}{args})')
+                        print(f"&Detector::{child.spelling}{args})")
                     cn.append(child)
+
     for child in node.get_children():
         visit(child)
 
@@ -139,16 +154,16 @@ if __name__ == "__main__":
         "-v",
         "--verbose",
         help="more output",
-        action='store_true',
+        action="store_true",
     )
     cargs = parser.parse_args()
-    
+
     check_clang_format_version(12)
     check_for_compile_commands_json(cargs.build_path)
 
-    print("Parsing functions in Detector.h - ", end = "", flush = True)
+    print("Parsing functions in Detector.h - ", end="", flush=True)
     t0 = time.perf_counter()
-    #parse functions
+    # parse functions
     db = cindex.CompilationDatabase.fromDirectory(cargs.build_path)
     index = cindex.Index.create()
     args = db.getCompileCommands(fpath)
@@ -159,10 +174,10 @@ if __name__ == "__main__":
     args = args + incargs
     tu = index.parse(fpath, args=args)
     visit(tu.cursor)
-    print(green('OK'))
-    print(f'Parsing took {time.perf_counter()-t0:.3f}s')
+    print(green("OK"))
+    print(f"Parsing took {time.perf_counter()-t0:.3f}s")
 
-    print("Read detector_in.cpp - ", end = "")
+    print("Read detector_in.cpp - ", end="")
     with open("../src/detector_in.cpp") as f:
         data = f.read()
     s = "".join(lines)
@@ -170,16 +185,16 @@ if __name__ == "__main__":
     text = data.replace("[[FUNCTIONS]]", s)
     warning = "/* WARINING This file is auto generated any edits might be overwritten without warning */\n\n"
     print(green("OK"))
-    print("Writing to detector.cpp - ", end = "")
+    print("Writing to detector.cpp - ", end="")
     with open("../src/detector.cpp", "w") as f:
         f.write(warning)
         f.write(text)
-    print(green('OK'))
+    print(green("OK"))
 
     # run clang format on the output
-    print('Running clang format on generated source -', end = "")
+    print("Running clang format on generated source -", end="")
     subprocess.run(["clang-format", "../src/detector.cpp", "-i"])
     print(green(" OK"))
 
     print("Changes since last commit:")
-    subprocess.run(['git',  'diff', '../src/detector.cpp'])
+    subprocess.run(["git", "diff", "../src/detector.cpp"])

python/slsdet/__init__.py

@@ -24,4 +24,5 @@ from .defines import *
 IpAddr = _slsdet.IpAddr
 MacAddr = _slsdet.MacAddr
 scanParameters = _slsdet.scanParameters
-currentSrcParameters = _slsdet.currentSrcParameters
+currentSrcParameters = _slsdet.currentSrcParameters
+DurationWrapper = _slsdet.DurationWrapper

python/slsdet/utils.py

@@ -79,7 +79,7 @@ def element_if_equal(mylist):
 
 def reduce_time(mylist):
     res = element_if_equal(element_if_equal(mylist))
-    if isinstance(res, dt.timedelta):
+    if isinstance(res, (dt.timedelta, _slsdet.DurationWrapper)):
         return res.total_seconds()
     elif isinstance(res[0], list):
         return [[item.total_seconds() for item in subl] for subl in res]

python/src/DurationWrapper.cpp

@@ -0,0 +1,26 @@
+#include "DurationWrapper.h"
+#include <cmath>
+
+namespace sls{
+
+    DurationWrapper::DurationWrapper(double seconds){
+        ns_tick = std::round(seconds*1e9);
+    }
+
+    uint64_t DurationWrapper::count() const{
+        return ns_tick;
+    }
+
+    void DurationWrapper::set_count(uint64_t ns_count){
+        ns_tick = ns_count;
+    }
+
+    bool DurationWrapper::operator==(const DurationWrapper& other)const{
+        return ns_tick == other.ns_tick;
+    }
+
+    double DurationWrapper::total_seconds()const{
+        return static_cast<double>(ns_tick)/1e9;
+    }
+
+}

python/src/DurationWrapper.h

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
+#pragma once
+#include <cstdint>
+
+namespace sls{
+/*
+Wrapper for nanoseconds stored in uint64_t, used for conversion between
+std::chrono::nanoseconds and python (float or sls::DurationWrapper)
+*/
+
+class DurationWrapper{
+    uint64_t ns_tick{0};
+
+    public:
+        DurationWrapper() = default;
+        explicit DurationWrapper(double seconds);
+        ~DurationWrapper() = default;
+
+        bool operator==(const DurationWrapper& other) const;
+        uint64_t count() const;
+        void set_count(uint64_t count);
+        double total_seconds() const; 
+};
+
+}

python/src/current.cpp

@@ -1,12 +1,8 @@
 // SPDX-License-Identifier: LGPL-3.0-or-other
 // Copyright (C) 2021 Contributors to the SLS Detector Package
-#include <pybind11/chrono.h>
-#include <pybind11/numpy.h>
-#include <pybind11/operators.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
 
-// #include "sls/Pattern.h"
+#include "py_headers.h"
+
 #include "sls/ToString.h"
 #include "sls/sls_detector_defs.h"
 

python/src/detector_in.cpp

@@ -1,17 +1,13 @@
 // SPDX-License-Identifier: LGPL-3.0-or-other
 // Copyright (C) 2021 Contributors to the SLS Detector Package
-#include <pybind11/chrono.h>
-#include <pybind11/operators.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include "py_headers.h"
 
 #include "sls/Detector.h"
 #include "sls/ToString.h"
 #include "sls/network_utils.h"
 #include "sls/sls_detector_defs.h"
-#include "typecaster.h"
-
 #include "sls/TimeHelper.h"
+
 #include <array>
 #include <chrono>
 namespace py = pybind11;
@@ -23,7 +19,7 @@ void init_det(py::module &m) {
     using sls::Result;
 
     py::class_<Detector> CppDetectorApi(m, "CppDetectorApi");
-    CppDetectorApi.def(py::init<int>())
+    CppDetectorApi.def(py::init<int>());
 
         [[FUNCTIONS]]
 }

python/src/duration.cpp

@@ -0,0 +1,21 @@
+#include "py_headers.h"
+
+#include "DurationWrapper.h"
+#include <sstream>
+namespace py = pybind11;
+using sls::DurationWrapper;
+
+void init_duration(py::module &m) {
+    py::class_<DurationWrapper>(m, "DurationWrapper")
+        .def(py::init())
+        .def(py::init<double>())
+        .def("total_seconds", &DurationWrapper::total_seconds)
+        .def("count", &DurationWrapper::count)
+        .def("set_count", &DurationWrapper::set_count)
+        .def("__repr__", [](const DurationWrapper &self) {
+            std::stringstream ss;
+            ss << "sls::DurationWrapper(total_seconds: " << self.total_seconds()
+               << " count: " << self.count() << ")";
+            return ss.str();
+        });
+}

python/src/enums.cpp

@@ -3,11 +3,7 @@
 
 // SPDX-License-Identifier: LGPL-3.0-or-other
 // Copyright (C) 2021 Contributors to the SLS Detector Package
-#include <pybind11/chrono.h>
-#include <pybind11/numpy.h>
-#include <pybind11/operators.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include "py_headers.h"
 
 #include "sls/Pattern.h"
 #include "sls/sls_detector_defs.h"

python/src/enums_in.cpp

@@ -1,10 +1,6 @@
 // SPDX-License-Identifier: LGPL-3.0-or-other
 // Copyright (C) 2021 Contributors to the SLS Detector Package
-#include <pybind11/chrono.h>
-#include <pybind11/numpy.h>
-#include <pybind11/operators.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include "py_headers.h"
 
 #include "sls/Pattern.h"
 #include "sls/sls_detector_defs.h"

python/src/main.cpp

@@ -1,9 +1,6 @@
 // SPDX-License-Identifier: LGPL-3.0-or-other
 // Copyright (C) 2021 Contributors to the SLS Detector Package
-#include <pybind11/chrono.h>
-#include <pybind11/operators.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include "py_headers.h"
 
 #include "mythenFileIO.h"
 #include "sls/Detector.h"
@@ -11,8 +8,6 @@
 #include <chrono>
 #include <vector>
 
-#include "typecaster.h"
-
 using ds = std::chrono::duration<double>;
 
 namespace py = pybind11;
@@ -23,6 +18,7 @@ void init_network(py::module &);
 void init_pattern(py::module &);
 void init_scan(py::module &);
 void init_source(py::module &);
+void init_duration(py::module &);
 PYBIND11_MODULE(_slsdet, m) {
     m.doc() = R"pbdoc(
         C/C++ API
@@ -40,6 +36,7 @@ PYBIND11_MODULE(_slsdet, m) {
     init_pattern(m);
     init_scan(m);
     init_source(m);
+    init_duration(m);
     //  init_experimental(m);
 
     py::module io = m.def_submodule("io", "Submodule for io");

python/src/mythenFileIO.h

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: LGPL-3.0-or-other
 // Copyright (C) 2021 Contributors to the SLS Detector Package
 #pragma once
+#include "py_headers.h"
 
 #include <cassert>
 #include <cmath>
@@ -9,9 +10,6 @@
 #include <iostream>
 #include <vector>
 
-#include <pybind11/numpy.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
 namespace py = pybind11;
 
 template <size_t bit_index0, size_t bit_index1>

python/src/network.cpp

@@ -4,11 +4,7 @@
 This file contains Python bindings for the IpAddr and MacAddr
 classes.
 */
-
-#include <pybind11/chrono.h>
-#include <pybind11/operators.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include "py_headers.h"
 
 #include "sls/network_utils.h"
 namespace py = pybind11;

python/src/pattern.cpp

@@ -1,10 +1,6 @@
 // SPDX-License-Identifier: LGPL-3.0-or-other
 // Copyright (C) 2021 Contributors to the SLS Detector Package
-#include <pybind11/chrono.h>
-#include <pybind11/numpy.h>
-#include <pybind11/operators.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
+#include "py_headers.h"
 
 #include "sls/Pattern.h"
 #include "sls/sls_detector_defs.h"

python/src/py_headers.h

@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2021 Contributors to the SLS Detector Package
+
+/* 
+Single common header file to make sure the pybind includes are the 
+same and ordered in the same way in all files. Needed to avoid
+ODR warnings
+*/
+#pragma once
+#include <pybind11/pybind11.h>
+#include <pybind11/operators.h>
+#include <pybind11/stl.h>
+#include <pybind11/numpy.h>
+#include "typecaster.h"

python/src/scan.cpp

@@ -1,11 +1,8 @@
 // SPDX-License-Identifier: LGPL-3.0-or-other
 // Copyright (C) 2021 Contributors to the SLS Detector Package
+#include "py_headers.h"
+
 #include "sls/sls_detector_defs.h"
-#include <pybind11/chrono.h>
-#include <pybind11/numpy.h>
-#include <pybind11/operators.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
 #include <sstream>
 namespace py = pybind11;
 void init_scan(py::module &m) {

python/src/typecaster.h

@@ -1,13 +1,92 @@
 // SPDX-License-Identifier: LGPL-3.0-or-other
 // Copyright (C) 2021 Contributors to the SLS Detector Package
 #pragma once
-#include "sls/Result.h"
 #include <pybind11/pybind11.h>
-// Add type_typecaster to pybind for our wrapper type
+#include <datetime.h>
+
+#include "sls/Result.h"
+#include "DurationWrapper.h"
+
+namespace py = pybind11;
 namespace pybind11 {
 namespace detail {
 template <typename Type, typename Alloc>
 struct type_caster<sls::Result<Type, Alloc>>
     : list_caster<sls::Result<Type, Alloc>, Type> {};
+
+
+// Based on the typecaster in pybind11/chrono.h
+template <> struct type_caster<std::chrono::nanoseconds> {
+    public:
+        PYBIND11_TYPE_CASTER(std::chrono::nanoseconds, const_name("DurationWrapper"));
+
+        // signed 25 bits required by the standard.
+        using days = std::chrono::duration<int_least32_t, std::ratio<86400>>;
+
+        /**
+         * Conversion part 1 (Python->C++): convert a PyObject into std::chrono::nanoseconds
+         * try datetime.timedelta, floats and our DurationWrapper wrapper
+         */
+
+        bool load(handle src, bool) {
+            using namespace std::chrono;
+
+            // Lazy initialise the PyDateTime import
+            if (!PyDateTimeAPI) {
+                PyDateTime_IMPORT;
+            }
+
+            if (!src) {
+                return false;
+            }
+            // If invoked with datetime.delta object, same as in chrono.h
+            if (PyDelta_Check(src.ptr())) {
+                value = duration_cast<nanoseconds>(
+                    days(PyDateTime_DELTA_GET_DAYS(src.ptr())) +
+                    seconds(PyDateTime_DELTA_GET_SECONDS(src.ptr())) +
+                    microseconds(PyDateTime_DELTA_GET_MICROSECONDS(src.ptr()))
+                    
+                );
+                return true;
+            }
+            // If invoked with a float we assume it is seconds and convert, same as in chrono.h
+            if (PyFloat_Check(src.ptr())) {
+                value = duration_cast<nanoseconds>(duration<double>(PyFloat_AsDouble(src.ptr())));
+                return true;
+            }
+
+
+            // Lastly if we were actually called with a DurationWrapper object we get
+            // the number of nanoseconds and create a std::chrono::nanoseconds from it
+            py::object py_cls = py::module::import("_slsdet").attr("DurationWrapper");
+            if (py::isinstance(src, py_cls)){
+                sls::DurationWrapper *cls = src.cast<sls::DurationWrapper *>();
+                value = nanoseconds(cls->count());
+                return true;
+            }
+            
+            
+            return false;
+
+        }
+
+        /**
+         * Conversion part 2 (C++ -> Python)
+         * import the module to get a handle to the wrapped class
+         * Default construct an object of (wrapped) DurationWrapper
+         * set the count from chrono::nanoseconds and return
+         */
+        static handle cast(std::chrono::nanoseconds src, return_value_policy /* policy */, handle /* parent */) {
+            py::object py_cls = py::module::import("_slsdet").attr("DurationWrapper");
+            py::object* obj = new py::object;
+            *obj = py_cls();
+            sls::DurationWrapper *dur = obj->cast<sls::DurationWrapper *>();
+            dur->set_count(src.count());
+            return *obj;
+        }
+    };
+
+
+
 } // namespace detail
 } // namespace pybind11