migrated tags for tests and added missing raw files (#206)

- No changes or evaluation of existing tests
- Tags for including tests that require data is changed to
**[.with-data]** and **--with-data** for C++ and python respectively
- Minor update to docs
- Added missing files to the test data repo

docs/src/Tests.rst

@@ -2,7 +2,7 @@
 Tests
 ****************
 
-We test the code both from the C++ and Python API. By default only tests that does not require image data is run. 
+We test the code both from C++ and Python. By default only tests that does not require additional data are run.
 
 C++
 ~~~~~~~~~~~~~~~~~~
@@ -15,7 +15,7 @@ C++
     make -j 4
 
     export AARE_TEST_DATA=/path/to/test/data
-    ./run_test [.files] #or using ctest, [.files] is the option to include tests needing data
+    ./run_test [.with-data] #or using ctest, [.with-data] is the option to include tests needing data
 
 
 
@@ -25,7 +25,7 @@ Python
 .. code-block:: bash
 
     #From the root dir of the library
-    python -m pytest python/tests --files # passing --files will run the tests needing data
+    python -m pytest python/tests --with-data # passing --with-data will run the tests needing data
 
 
 
@@ -35,7 +35,7 @@ Getting the test data
 .. attention ::
 
     The tests needing the test data are not run by default. To make the data available, you need to set the environment variable
-    AARE_TEST_DATA to the path of the test data directory. Then pass either [.files] for the C++ tests or --files for Python
+    AARE_TEST_DATA to the path of the test data directory. Then pass either [.with-data] for the C++ tests or --files for Python
 
 The image files needed for the test are large and are not included in the repository. They are stored
 using GIT LFS in a separate repository. To get the test data, you need to clone the repository.

python/tests/conftest.py

@@ -6,20 +6,20 @@ import pytest
 
 def pytest_addoption(parser):
     parser.addoption(
-        "--files", action="store_true", default=False, help="run slow tests"
+        "--with-data", action="store_true", default=False, help="Run tests that require additional data"
     )
 
 
 def pytest_configure(config):
-    config.addinivalue_line("markers", "files: mark test as needing image files to run")
+    config.addinivalue_line("markers", "withdata: mark test as needing image files to run")
 
 
 def pytest_collection_modifyitems(config, items):
-    if config.getoption("--files"):
+    if config.getoption("--with-data"):
         return
-    skip = pytest.mark.skip(reason="need --files option to run")
+    skip = pytest.mark.skip(reason="need --with-data option to run")
     for item in items:
-        if "files" in item.keywords:
+        if "withdata" in item.keywords:
             item.add_marker(skip)
 
 

python/tests/test_ClusterFile.py

@@ -9,7 +9,7 @@ import pickle
 from aare import ClusterFile
 from conftest import test_data_path
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_cluster_file(test_data_path): 
     """Test ClusterFile""" 
     f =  ClusterFile(test_data_path / "clust/single_frame_97_clustrers.clust") 
@@ -39,7 +39,7 @@ def test_cluster_file(test_data_path):
     for i in range(10):
         assert arr[i]['x'] == i+1
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_read_clusters_and_fill_histogram(test_data_path): 
     # Create the histogram
     n_bins = 100

python/tests/test_RawFile.py

@@ -2,7 +2,7 @@ import pytest
 from aare import RawFile
 import numpy as np
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_read_rawfile_with_roi(test_data_path):
 
    with RawFile(test_data_path / "raw/SingleChipROI/Data_master_0.json") as f:
@@ -11,7 +11,7 @@ def test_read_rawfile_with_roi(test_data_path):
     assert headers.size == 10100
     assert frames.shape == (10100, 256, 256)
     
-@pytest.mark.files
+@pytest.mark.withdata
 def test_read_rawfile_quad_eiger_and_compare_to_numpy(test_data_path): 
     
     d0 = test_data_path/'raw/eiger_quad_data/W13_vrpreampscan_m21C_300V_800eV_vthre2000_d0_f0_0.raw'
@@ -36,7 +36,7 @@ def test_read_rawfile_quad_eiger_and_compare_to_numpy(test_data_path):
     assert (image == image1).all()
 
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_read_rawfile_eiger_and_compare_to_numpy(test_data_path): 
     d0 = test_data_path/'raw/eiger/Lab6_20500eV_2deg_20240629_d0_f0_7.raw'
     d1 = test_data_path/'raw/eiger/Lab6_20500eV_2deg_20240629_d1_f0_7.raw'

python/tests/test_RawSubFile.py

@@ -3,7 +3,7 @@ import numpy as np
 from aare import RawSubFile, DetectorType
 
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_read_a_jungfrau_RawSubFile(test_data_path):
 
     # Starting with f1 there is now 7 frames left in the series of files
@@ -23,7 +23,7 @@ def test_read_a_jungfrau_RawSubFile(test_data_path):
     data = np.load(test_data_path / "raw/jungfrau/jungfrau_single_0.npy")
     assert np.all(data[3:] == frames)
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_iterate_over_a_jungfrau_RawSubFile(test_data_path):
 
     data = np.load(test_data_path / "raw/jungfrau/jungfrau_single_0.npy")

python/tests/test_jungfrau_dat_files.py

@@ -2,7 +2,7 @@ import pytest
 import numpy as np
 from aare import JungfrauDataFile
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_jfungfrau_dat_read_number_of_frames(test_data_path):
     with JungfrauDataFile(test_data_path / "dat/AldoJF500k_000000.dat") as dat_file:
         assert dat_file.total_frames == 24
@@ -14,7 +14,7 @@ def test_jfungfrau_dat_read_number_of_frames(test_data_path):
         assert dat_file.total_frames == 113
 
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_jfungfrau_dat_read_number_of_file(test_data_path):
     with JungfrauDataFile(test_data_path / "dat/AldoJF500k_000000.dat") as dat_file:
         assert dat_file.n_files == 4
@@ -26,7 +26,7 @@ def test_jfungfrau_dat_read_number_of_file(test_data_path):
         assert dat_file.n_files == 7
 
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_read_module(test_data_path):
     """
     Read all frames from the series of .dat files. Compare to canned data in npz format. 
@@ -50,7 +50,7 @@ def test_read_module(test_data_path):
     assert np.all(ref_header == header)
     assert np.all(ref_data == data)
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_read_half_module(test_data_path):
 
     # Read all frames from the .dat file
@@ -71,7 +71,7 @@ def test_read_half_module(test_data_path):
     assert np.all(ref_data == data)
 
 
-@pytest.mark.files
+@pytest.mark.withdata
 def test_read_single_chip(test_data_path):
 
     # Read all frames from the .dat file

src/ClusterFile.test.cpp

@@ -10,7 +10,7 @@ using aare::Cluster;
 using aare::ClusterFile;
 using aare::ClusterVector;
 
-TEST_CASE("Read one frame from a cluster file", "[.files]") {
+TEST_CASE("Read one frame from a cluster file", "[.with-data]") {
     // We know that the frame has 97 clusters
     auto fpath = test_data_path() / "clust" / "single_frame_97_clustrers.clust";
     REQUIRE(std::filesystem::exists(fpath));
@@ -26,7 +26,7 @@ TEST_CASE("Read one frame from a cluster file", "[.files]") {
                      std::begin(expected_cluster_data)));
 }
 
-TEST_CASE("Read one frame using ROI", "[.files]") {
+TEST_CASE("Read one frame using ROI", "[.with-data]") {
     // We know that the frame has 97 clusters
     auto fpath = test_data_path() / "clust" / "single_frame_97_clustrers.clust";
     REQUIRE(std::filesystem::exists(fpath));
@@ -58,7 +58,7 @@ TEST_CASE("Read one frame using ROI", "[.files]") {
                      std::begin(expected_cluster_data)));
 }
 
-TEST_CASE("Read clusters from single frame file", "[.files]") {
+TEST_CASE("Read clusters from single frame file", "[.with-data]") {
 
     //  frame_number, num_clusters   [135] 97
     // [  1 200] [0 1 2 3 4 5 6 7 8]
@@ -202,7 +202,7 @@ TEST_CASE("Read clusters from single frame file", "[.files]") {
     }
 }
 
-TEST_CASE("Read clusters from single frame file with ROI", "[.files]") {
+TEST_CASE("Read clusters from single frame file with ROI", "[.with-data]") {
     auto fpath = test_data_path() / "clust" / "single_frame_97_clustrers.clust";
     REQUIRE(std::filesystem::exists(fpath));
 
@@ -226,7 +226,7 @@ TEST_CASE("Read clusters from single frame file with ROI", "[.files]") {
                      std::begin(expected_cluster_data)));
 }
 
-TEST_CASE("Read cluster from multiple frame file", "[.files]") {
+TEST_CASE("Read cluster from multiple frame file", "[.with-data]") {
 
     using ClusterType = Cluster<double, 2, 2>;
 
@@ -279,7 +279,7 @@ TEST_CASE("Read cluster from multiple frame file", "[.files]") {
     }
 }
 
-TEST_CASE("Write cluster with potential padding", "[.files][.ClusterFile]") {
+TEST_CASE("Write cluster with potential padding", "[.with-data][.ClusterFile]") {
 
     using ClusterType = Cluster<double, 3, 3>;
 
@@ -324,7 +324,7 @@ TEST_CASE("Write cluster with potential padding", "[.files][.ClusterFile]") {
         }));
 }
 
-TEST_CASE("Read frame and modify cluster data", "[.files][.ClusterFile]") {
+TEST_CASE("Read frame and modify cluster data", "[.with-data][.ClusterFile]") {
     auto fpath = test_data_path() / "clust" / "single_frame_97_clustrers.clust";
     REQUIRE(std::filesystem::exists(fpath));
 

src/ClusterFinderMT.test.cpp

@@ -57,9 +57,9 @@ class ClusterFinderMTWrapper
     size_t m_sink_size() const { return this->m_sink.sizeGuess(); }
 };
 
-TEST_CASE("multithreaded cluster finder", "[.files][.ClusterFinder]") {
+TEST_CASE("multithreaded cluster finder", "[.with-data]") {
     auto fpath =
-        test_data_path() / "clust/Moench03new/cu_half_speed_master_4.json";
+        test_data_path() / "raw/moench03/cu_half_speed_master_4.json";
 
     REQUIRE(std::filesystem::exists(fpath));
 

src/JungfrauDataFile.test.cpp

@@ -5,7 +5,7 @@
 
 using aare::JungfrauDataFile;
 using aare::JungfrauDataHeader;
-TEST_CASE("Open a Jungfrau data file", "[.files]") {
+TEST_CASE("Open a Jungfrau data file", "[.with-data]") {
     // we know we have 4 files with 7, 7, 7, and 3 frames
     // firs frame number if 1 and the bunch id is frame_number**2
     // so we can check the header
@@ -37,7 +37,7 @@ TEST_CASE("Open a Jungfrau data file", "[.files]") {
     }
 }
 
-TEST_CASE("Seek in a JungfrauDataFile", "[.files]") {
+TEST_CASE("Seek in a JungfrauDataFile", "[.with-data]") {
     auto fpath = test_data_path() / "dat" / "AldoJF65k_000000.dat";
     REQUIRE(std::filesystem::exists(fpath));
 
@@ -70,7 +70,7 @@ TEST_CASE("Seek in a JungfrauDataFile", "[.files]") {
     REQUIRE_THROWS(f.seek(86356)); // out of range
 }
 
-TEST_CASE("Open a Jungfrau data file with non zero file index", "[.files]") {
+TEST_CASE("Open a Jungfrau data file with non zero file index", "[.with-data]") {
 
     auto fpath = test_data_path() / "dat" / "AldoJF65k_000003.dat";
     REQUIRE(std::filesystem::exists(fpath));
@@ -94,7 +94,7 @@ TEST_CASE("Open a Jungfrau data file with non zero file index", "[.files]") {
     REQUIRE(f.current_file().stem() == "AldoJF65k_000003");
 }
 
-TEST_CASE("Read into throws if size doesn't match", "[.files]") {
+TEST_CASE("Read into throws if size doesn't match", "[.with-data]") {
     auto fpath = test_data_path() / "dat" / "AldoJF65k_000000.dat";
     REQUIRE(std::filesystem::exists(fpath));
 

src/NumpyFile.test.cpp

@@ -6,7 +6,7 @@
 
 using aare::Dtype;
 using aare::NumpyFile;
-TEST_CASE("Read a 1D numpy file with int32 data type", "[.integration]") {
+TEST_CASE("Read a 1D numpy file with int32 data type", "[.with-data]") {
 
     auto fpath = test_data_path() / "numpy" / "test_1d_int32.npy";
     REQUIRE(std::filesystem::exists(fpath));
@@ -24,7 +24,7 @@ TEST_CASE("Read a 1D numpy file with int32 data type", "[.integration]") {
     }
 }
 
-TEST_CASE("Read a 3D numpy file with np.double data type", "[.integration]") {
+TEST_CASE("Read a 3D numpy file with np.double data type", "[.with-data]") {
 
     auto fpath = test_data_path() / "numpy" / "test_3d_double.npy";
     REQUIRE(std::filesystem::exists(fpath));

src/RawFile.test.cpp

@@ -13,19 +13,19 @@ using aare::File;
 using aare::RawFile;
 using namespace aare;
 
-TEST_CASE("Read number of frames from a jungfrau raw file", "[.integration]") {
+TEST_CASE("Read number of frames from a jungfrau raw file", "[.with-data]") {
 
     auto fpath =
-        test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+        test_data_path() / "raw/jungfrau/jungfrau_single_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
 
     File f(fpath, "r");
     REQUIRE(f.total_frames() == 10);
 }
 
-TEST_CASE("Read frame numbers from a jungfrau raw file", "[.integration]") {
+TEST_CASE("Read frame numbers from a jungfrau raw file", "[.with-data]") {
     auto fpath =
-        test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+        test_data_path() / "raw/jungfrau/jungfrau_single_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
 
     File f(fpath, "r");
@@ -40,9 +40,9 @@ TEST_CASE("Read frame numbers from a jungfrau raw file", "[.integration]") {
     }
 }
 
-TEST_CASE("Read a frame number too high throws", "[.integration]") {
+TEST_CASE("Read a frame number too high throws", "[.with-data]") {
     auto fpath =
-        test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+        test_data_path() / "raw/jungfrau/jungfrau_single_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
 
     File f(fpath, "r");
@@ -56,9 +56,10 @@ TEST_CASE("Read a frame number too high throws", "[.integration]") {
 }
 
 TEST_CASE("Read a frame numbers where the subfile is missing throws",
-          "[.integration]") {
+          "[.with-data]") {
-    auto fpath = test_data_path() / "jungfrau" /
+    auto fpath = test_data_path() / "raw/jungfrau" /
                  "jungfrau_missing_subfile_master_0.json";
+
     REQUIRE(std::filesystem::exists(fpath));
 
     File f(fpath, "r");
@@ -78,9 +79,9 @@ TEST_CASE("Read a frame numbers where the subfile is missing throws",
 }
 
 TEST_CASE("Read data from a jungfrau 500k single port raw file",
-          "[.integration]") {
+          "[.with-data]") {
     auto fpath =
-        test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+        test_data_path() / "raw/jungfrau/jungfrau_single_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
 
     File f(fpath, "r");
@@ -97,8 +98,8 @@ TEST_CASE("Read data from a jungfrau 500k single port raw file",
     }
 }
 
-TEST_CASE("Read frame numbers from a raw file", "[.integration]") {
+TEST_CASE("Read frame numbers from a raw file", "[.with-data]") {
-    auto fpath = test_data_path() / "eiger" / "eiger_500k_16bit_master_0.json";
+    auto fpath = test_data_path() / "raw/eiger" / "eiger_500k_16bit_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
 
     // we know this file has 3 frames with frame numbers 14, 15, 16
@@ -110,7 +111,7 @@ TEST_CASE("Read frame numbers from a raw file", "[.integration]") {
     }
 }
 
-TEST_CASE("Compare reading from a numpy file with a raw file", "[.files]") {
+TEST_CASE("Compare reading from a numpy file with a raw file", "[.with-data]") {
 
     SECTION("jungfrau data") {
         auto fpath_raw =
@@ -173,9 +174,9 @@ TEST_CASE("Compare reading from a numpy file with a raw file", "[.files]") {
     }
 }
 
-TEST_CASE("Read multipart files", "[.integration]") {
+TEST_CASE("Read multipart files", "[.with-data]") {
     auto fpath =
-        test_data_path() / "jungfrau" / "jungfrau_double_master_0.json";
+        test_data_path() / "raw/jungfrau" / "jungfrau_double_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
 
     File f(fpath, "r");
@@ -214,7 +215,7 @@ struct TestParameters {
     std::vector<ModuleGeometry> module_geometries{};
 };
 
-TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") {
+TEST_CASE("check find_geometry", "[.with-data]") {
 
     auto test_parameters = GENERATE(
         TestParameters{"raw/jungfrau_2modules_version6.1.2/run_master_0.raw", 2,
@@ -288,7 +289,7 @@ TEST_CASE("check find_geometry", "[.integration][.files][.rawfile]") {
 }
 
 TEST_CASE("Open multi module file with ROI",
-          "[.integration][.files][.rawfile]") {
+          "[.with-data]") {
 
     auto fpath = test_data_path() / "raw/SingleChipROI/Data_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
@@ -312,9 +313,9 @@ TEST_CASE("Open multi module file with ROI",
     }
 }
 
-TEST_CASE("Read file with unordered frames", "[.integration]") {
+TEST_CASE("Read file with unordered frames", "[.with-data]") {
     // TODO! Better explanation and error message
-    auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw";
+    auto fpath = test_data_path() / "raw/mythen/scan242_master_3.raw";
     REQUIRE(std::filesystem::exists(fpath));
     File f(fpath);
     REQUIRE_THROWS((f.read_frame()));

src/RawMasterFile.test.cpp

@@ -147,7 +147,7 @@ TEST_CASE("Parse a master file in .json format", "[.integration]") {
 }
 
 TEST_CASE("Parse a master file in old .raw format",
-          "[.integration][.files][.rawmasterfile]") {
+          "[.integration][.with-data][.rawmasterfile]") {
     auto fpath = test_data_path() /
                  "raw/jungfrau_2modules_version6.1.2/run_master_0.raw";
     REQUIRE(std::filesystem::exists(fpath));

src/RawSubFile.test.cpp

@@ -6,7 +6,7 @@
 
 using namespace aare;
 
-TEST_CASE("Read frames directly from a RawSubFile", "[.files]") {
+TEST_CASE("Read frames directly from a RawSubFile", "[.with-data]") {
     auto fpath_raw =
         test_data_path() / "raw/jungfrau" / "jungfrau_single_d0_f0_0.raw";
     REQUIRE(std::filesystem::exists(fpath_raw));
@@ -40,7 +40,7 @@ TEST_CASE("Read frames directly from a RawSubFile", "[.files]") {
 }
 
 TEST_CASE("Read frames directly from a RawSubFile starting at the second file",
-          "[.files]") {
+          "[.with-data]") {
     // we know this file has 10 frames with frame numbers 1 to 10
     // f0 1,2,3
     // f1 4,5,6 <-- starting here