docs and python tests

docs/src/JungfrauDataFile.rst

@@ -0,0 +1,8 @@
+JungfrauDataFile
+==================
+
+
+.. doxygenclass:: aare::JungfrauDataFile
+   :members:
+   :undoc-members:
+   :private-members:

docs/src/Tests.rst

@@ -0,0 +1,47 @@
+****************
+Tests
+****************
+
+We test the code both from the C++ and Python API. By default only tests that does not require image data is run. 
+
+C++
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    mkdir build
+    cd build
+    cmake .. -DAARE_TESTS=ON
+    make -j 4
+
+    export AARE_TEST_DATA_DIR=/path/to/test/data
+    ./run_test [.files] #or using ctest, [.files] is the option to include tests needing data
+
+
+
+Python
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    #From the root dir of the library
+    python -m pytest python/tests --files # passing --files will run the tests needing data
+
+
+
+Getting the test data
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. attention ::
+
+    The tests needing the test data are not run by default. To make the data available, you need to set the environment variable
+    AARE_TEST_DATA to the path of the test data directory. Then pass either [.files] for the C++ tests or --files for Python
+
+The image files needed for the test are large and are not included in the repository. They are stored
+using GIT LFS in a separate repository. To get the test data, you need to clone the repository.
+To do this, you need to have GIT LFS installed. You can find instructions on how to install it here: https://git-lfs.github.com/
+Once you have GIT LFS installed, you can clone the repository like any normal repo using:
+
+.. code-block:: bash
+    
+    git clone https://gitea.psi.ch/detectors/aare-test-data.git

docs/src/index.rst

@@ -20,9 +20,6 @@ AARE
     Requirements
     Consume
 
-
-
-
 .. toctree::
     :caption: Python API
     :maxdepth: 1
@@ -31,6 +28,7 @@ AARE
     pyCtbRawFile
     pyClusterFile
     pyClusterVector
+    pyJungfrauDataFile
     pyRawFile
     pyRawMasterFile
     pyVarClusterFinder
@@ -51,6 +49,7 @@ AARE
     ClusterFinderMT
     ClusterFile
     ClusterVector
+    JungfrauDataFile
     Pedestal
     RawFile
     RawSubFile
@@ -59,4 +58,8 @@ AARE
 
 
 
-
+.. toctree::
+    :caption: Developer
+    :maxdepth: 3
+    
+    Tests

docs/src/pyJungfrauDataFile.rst

@@ -0,0 +1,10 @@
+JungfrauDataFile
+===================
+
+.. py:currentmodule:: aare
+
+.. autoclass:: JungfrauDataFile
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

pyproject.toml

@@ -15,4 +15,9 @@ cmake.verbose = true
 [tool.scikit-build.cmake.define]
 AARE_PYTHON_BINDINGS = "ON"
 AARE_SYSTEM_LIBRARIES = "ON"
-AARE_INSTALL_PYTHONEXT = "ON"
+AARE_INSTALL_PYTHONEXT = "ON"
+
+[tool.pytest.ini_options]
+markers = [
+    "files: marks tests that need additional data (deselect with '-m \"not files\"')",
+]

python/src/jungfrau_data_file.hpp

@@ -2,7 +2,6 @@
 #include "aare/JungfrauDataFile.hpp"
 #include "aare/defs.hpp"
 
-
 #include <cstdint>
 #include <filesystem>
 #include <pybind11/iostream.h>
@@ -15,8 +14,48 @@
 namespace py = pybind11;
 using namespace ::aare;
 
+// Disable warnings for unused parameters, as we ignore some
+// in the __exit__ method
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+
+auto read_dat_frame(JungfrauDataFile &self) {
+    std::vector<ssize_t> shape;
+    shape.reserve(2);
+    shape.push_back(self.rows());
+    shape.push_back(self.cols());
+
+    // return headers from all subfiles
+    py::array_t<JungfrauDataHeader> header(1);
+    py::array_t<uint16_t> image(shape);
+
+    self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()),
+                   header.mutable_data());
+
+    return py::make_tuple(header, image);
+}
+
+auto read_n_dat_frames(JungfrauDataFile &self, size_t n_frames) {
+    // adjust for actual frames left in the file
+    n_frames = std::min(n_frames, self.total_frames() - self.tell());
+    if (n_frames == 0) {
+        throw std::runtime_error("No frames left in file");
+    }
+    std::vector<size_t> shape{n_frames, self.rows(), self.cols()};
+
+    // return headers from all subfiles
+    py::array_t<JungfrauDataHeader> header(n_frames);
+
+    py::array_t<uint16_t> image(shape);
+
+    self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()),
+                   n_frames, header.mutable_data());
+
+    return py::make_tuple(header, image);
+}
+
 void define_jungfrau_data_file_io_bindings(py::module &m) {
-    //Make the JungfrauDataHeader usable from numpy
+    // Make the JungfrauDataHeader usable from numpy
     PYBIND11_NUMPY_DTYPE(JungfrauDataHeader, framenum, bunchid);
 
     py::class_<JungfrauDataFile>(m, "JungfrauDataFile")
@@ -33,50 +72,44 @@ void define_jungfrau_data_file_io_bindings(py::module &m) {
         .def_property_readonly("bytes_per_pixel",
                                &JungfrauDataFile::bytes_per_pixel)
         .def_property_readonly("bitdepth", &JungfrauDataFile::bitdepth)
-        .def_property_readonly("current_file",
-                               &JungfrauDataFile::current_file)
-        .def_property_readonly("total_frames",
-                               &JungfrauDataFile::total_frames)
+        .def_property_readonly("current_file", &JungfrauDataFile::current_file)
+        .def_property_readonly("total_frames", &JungfrauDataFile::total_frames)
         .def_property_readonly("n_files", &JungfrauDataFile::n_files)
         .def("read_frame",
-            [](JungfrauDataFile &self) {
-                
-                std::vector<ssize_t> shape;
-                shape.reserve(2);
-                shape.push_back(self.rows());
-                shape.push_back(self.cols());
-
-                // return headers from all subfiles
-                py::array_t<JungfrauDataHeader> header(1);
-                py::array_t<uint16_t> image(shape);
-
-                self.read_into(
-                    reinterpret_cast<std::byte *>(image.mutable_data()),
-                    header.mutable_data());
-
-                return py::make_tuple(header, image);
-            })
+             [](JungfrauDataFile &self) { return read_dat_frame(self); })
         .def("read_n",
-            [](JungfrauDataFile &self, size_t n_frames) {
-                // adjust for actual frames left in the file
-                n_frames =
-                    std::min(n_frames, self.total_frames() - self.tell());
-                if (n_frames == 0) {
-                    throw std::runtime_error("No frames left in file");
-                }
-                std::vector<size_t> shape{n_frames, self.rows(), self.cols()};
+             [](JungfrauDataFile &self, size_t n_frames) {
+                 return read_n_dat_frames(self, n_frames);
+             },
+             R"(
+               Read maximum n_frames frames from the file.
+               )")
+        .def(
+            "read",
+            [](JungfrauDataFile &self) {
+                self.seek(0);
+                auto n_frames = self.total_frames();
+                return read_n_dat_frames(self, n_frames);
+            },
+            R"(
+              Read all frames from the file. Seeks to the beginning before reading.
+              )")
+        .def("__enter__", [](JungfrauDataFile &self) { return &self; })
+        .def("__exit__",
+             [](JungfrauDataFile &self,
+                const std::optional<pybind11::type> &exc_type,
+                const std::optional<pybind11::object> &exc_value,
+                const std::optional<pybind11::object> &traceback) {
+                 //  self.close();
+             })
+        .def("__iter__", [](JungfrauDataFile &self) { return &self; })
+        .def("__next__", [](JungfrauDataFile &self) {
+            try {
+                return read_dat_frame(self);
+            } catch (std::runtime_error &e) {
+                throw py::stop_iteration();
+            }
+        });
+}
 
-                // return headers from all subfiles
-                py::array_t<JungfrauDataHeader> header(n_frames);
-
-                py::array_t<uint16_t> image(shape);
-
-                self.read_into(
-                    reinterpret_cast<std::byte *>(image.mutable_data()),
-                    n_frames,
-                    header.mutable_data());
-
-                return py::make_tuple(header, image);
-            });
-
-}
+#pragma GCC diagnostic pop

python/tests/conftest.py

@@ -0,0 +1,29 @@
+import os
+from pathlib import Path
+import pytest
+
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--files", action="store_true", default=False, help="run slow tests"
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "files: mark test as needing image fiels to run")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--files"):
+        return
+    skip = pytest.mark.skip(reason="need --files option to run")
+    for item in items:
+        if "files" in item.keywords:
+            item.add_marker(skip)
+
+
+@pytest.fixture
+def test_data_path():
+    return Path(os.environ["AARE_TEST_DATA"])
+

python/tests/test_jungfrau_dat_files.py

@@ -0,0 +1,92 @@
+import pytest
+import numpy as np
+from aare import JungfrauDataFile
+
+@pytest.mark.files
+def test_jfungfrau_dat_read_number_of_frames(test_data_path):
+    with JungfrauDataFile(test_data_path / "dat/AldoJF500k_000000.dat") as dat_file:
+        assert dat_file.total_frames == 24
+
+    with JungfrauDataFile(test_data_path / "dat/AldoJF250k_000000.dat") as dat_file:
+        assert dat_file.total_frames == 53
+    
+    with JungfrauDataFile(test_data_path / "dat/AldoJF65k_000000.dat") as dat_file:
+        assert dat_file.total_frames == 113
+
+
+@pytest.mark.files
+def test_jfungfrau_dat_read_number_of_file(test_data_path):
+    with JungfrauDataFile(test_data_path / "dat/AldoJF500k_000000.dat") as dat_file:
+        assert dat_file.n_files == 4
+
+    with JungfrauDataFile(test_data_path / "dat/AldoJF250k_000000.dat") as dat_file:
+        assert dat_file.n_files == 7
+    
+    with JungfrauDataFile(test_data_path / "dat/AldoJF65k_000000.dat") as dat_file:
+        assert dat_file.n_files == 7
+
+
+@pytest.mark.files
+def test_read_module(test_data_path):
+    """
+    Read all frames from the series of .dat files. Compare to canned data in npz format. 
+    """
+
+    # Read all frames from the .dat file
+    with JungfrauDataFile(test_data_path / "dat/AldoJF500k_000000.dat") as f:
+        header, data = f.read()
+
+    #Sanity check
+    n_frames = 24
+    assert header.size == n_frames
+    assert data.shape == (n_frames, 512, 1024)
+
+    # Read reference data using numpy
+    with np.load(test_data_path / "dat/AldoJF500k.npz") as f:
+        ref_header = f["headers"]
+        ref_data = f["frames"]
+
+    # Check that the data is the same
+    assert np.all(ref_header == header)
+    assert np.all(ref_data == data)
+
+@pytest.mark.files
+def test_read_half_module(test_data_path):
+
+    # Read all frames from the .dat file
+    with JungfrauDataFile(test_data_path / "dat/AldoJF250k_000000.dat") as f:
+        header, data = f.read()
+
+    n_frames = 53
+    assert header.size == n_frames
+    assert data.shape == (n_frames, 256, 1024)
+
+    # Read reference data using numpy
+    with np.load(test_data_path / "dat/AldoJF250k.npz") as f:
+        ref_header = f["headers"]
+        ref_data = f["frames"]
+
+    # Check that the data is the same
+    assert np.all(ref_header == header)
+    assert np.all(ref_data == data)
+
+
+@pytest.mark.files
+def test_read_single_chip(test_data_path):
+
+    # Read all frames from the .dat file
+    with JungfrauDataFile(test_data_path / "dat/AldoJF65k_000000.dat") as f:
+        header, data = f.read()
+
+    n_frames = 113
+    assert header.size == n_frames
+    assert data.shape == (n_frames, 256, 256)
+
+    # Read reference data using numpy
+    with np.load(test_data_path / "dat/AldoJF65k.npz") as f:
+        ref_header = f["headers"]
+        ref_data = f["frames"]
+
+    # Check that the data is the same
+    assert np.all(ref_header == header)
+    assert np.all(ref_data == data)