added custom io for my302

- Build and run tests in CI
- Added macOS builds (and tests)
- Renamed workflow to build_with_docs.yml

.github/workflows/build_and_deploy_conda.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest, ] # macos-12, windows-2019]
+        platform: [ubuntu-latest] # macos-12, windows-2019]
         python-version: ["3.12",]
 
     runs-on: ${{ matrix.platform }}

.github/workflows/build_with_docs.yml

@@ -2,11 +2,15 @@ name: Build the package using cmake then documentation
 
 on:
   workflow_dispatch:
+  push:
   pull_request:
   release:
     types:
       - published
     
+env:
+  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
+  BUILD_TYPE: Debug
 
 permissions:
   contents: read
@@ -18,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest, ] 
+        platform: [ubuntu-latest, macos-latest] 
         python-version: ["3.12",]
 
     runs-on: ${{ matrix.platform }}
@@ -39,15 +43,20 @@ jobs:
           channels: conda-forge
           conda-remove-defaults: "true"
 
-      - name: Build library 
+      - name: Build library and docs
         run: |
             mkdir build 
             cd build 
-            cmake .. -DAARE_SYSTEM_LIBRARIES=ON -DAARE_PYTHON_BINDINGS=ON -DAARE_DOCS=ON 
-            make -j 2 
+            cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DAARE_SYSTEM_LIBRARIES=ON -DAARE_PYTHON_BINDINGS=ON -DAARE_DOCS=ON -DAARE_TESTS=ON
+            make -j 4 
             make docs
 
+      - name: C++ unit tests
+        working-directory: ${{github.workspace}}/build
+        run: ctest -C ${{env.BUILD_TYPE}} -j4
+
       - name: Upload static files as artifact
+        if: matrix.platform == 'ubuntu-latest'
         id: deployment
         uses: actions/upload-pages-artifact@v3 
         with:

etc/dev-env.yml

@@ -3,6 +3,7 @@ channels:
   - conda-forge
 dependencies:
   - anaconda-client
+  - catch2
   - conda-build
   - doxygen 
   - sphinx

include/aare/RawMasterFile.hpp

@@ -103,6 +103,7 @@ class RawMasterFile {
     std::optional<size_t> m_digital_samples;
     std::optional<size_t> m_transceiver_samples;
     std::optional<size_t> m_number_of_rows;
+    std::optional<uint8_t> m_counter_mask;
 
     std::optional<ROI> m_roi;
 
@@ -133,6 +134,7 @@ class RawMasterFile {
     std::optional<size_t> digital_samples() const;
     std::optional<size_t> transceiver_samples() const;
     std::optional<size_t> number_of_rows() const;
+    std::optional<uint8_t> counter_mask() const;
 
     std::optional<ROI> roi() const;
 

python/CMakeLists.txt

@@ -39,6 +39,8 @@ set( PYTHON_FILES
     aare/transform.py
     aare/ScanParameters.py
     aare/utils.py
+    aare/experimental/__init__.py
+    aare/experimental/custom_io.py
 )
 
 

python/aare/experimental/custom_io.py

@@ -0,0 +1,58 @@
+
+import numpy as np
+
+
+n_counters = 64*3
+bitfield_size = 64
+
+header_dt = [('frameNumber',np.uint64),
+             ('expLength',np.uint32),
+             ('packetNumber', np.uint32),
+             ('bunchId', np.uint64),
+             ('timestamp', np.uint64),
+             ('modId', np.uint16),
+             ('row', np.uint16),
+             ('col', np.uint16),
+             ('reserved', np.uint16),
+             ('debug', np.uint32),
+             ('roundRNumber', np.uint16),
+             ('detType', np.uint8),
+             ('version', np.uint8)]
+
+
+
+
+def ExtractBits(raw_data, dr=24, bits = (17,6)):
+    bits = np.uint64(bits)
+    data = np.zeros(0, dtype = np.uint64)
+    for bit in bits:
+        tmp = (raw_data >> bit) & np.uint64(1)
+        data = np.hstack((data, tmp))
+    
+    #Shift the bits to the righ place
+    for i in np.arange(dr, dtype = np.uint64):
+        data[i::dr] = data[i::dr] << i
+
+    data = data.reshape(data.size//dr, dr)
+    return data.sum(axis = 1)
+
+
+
+
+def read_my302_file(fname, dr=24, bits = (17,6),
+                    offset=48, tail = 72, n_frames=1):
+    header = np.zeros(n_frames, header_dt)
+    data = np.zeros((n_frames, n_counters), dtype = np.uint64)
+    with open(fname, 'rb') as f:
+        for i in range(n_frames):
+            header[i], raw_data = _read_my302_frame(f, offset, tail, dr)
+            data[i] = ExtractBits(raw_data, dr=dr, bits = bits)
+    return header, data
+
+
+def _read_my302_frame(f, offset, tail, dr):
+    header = np.fromfile(f, count=1, dtype = header_dt)
+    f.seek(bitfield_size+offset, 1)
+    data = np.fromfile(f, count = int(n_counters*dr/2), dtype = np.uint64)
+    f.seek(tail, 1)
+    return header, data

src/RawFile.test.cpp

@@ -320,4 +320,14 @@ TEST_CASE("Read file with unordered frames", "[.with-data]") {
     REQUIRE(std::filesystem::exists(fpath));
     File f(fpath);
     REQUIRE_THROWS((f.read_frame()));
+}
+
+TEST_CASE("Read Mythenframe", "[.with-data]") {
+    auto fpath = test_data_path() / "raw/newmythen03/run_2_master_1.json";
+    REQUIRE(std::filesystem::exists(fpath));
+    RawFile f(fpath);
+    REQUIRE(f.master().roi().value().width() == 2560);
+    REQUIRE(f.master().roi().value().height() == 1);
+    auto frame = f.read_frame();
+    REQUIRE(frame.cols() == 2560);
 }

src/RawMasterFile.cpp

@@ -101,7 +101,8 @@ int64_t ScanParameters::settleTime() const { return m_settleTime; }
 RawMasterFile::RawMasterFile(const std::filesystem::path &fpath)
     : m_fnc(fpath) {
     if (!std::filesystem::exists(fpath)) {
-        throw std::runtime_error(LOCATION + " File does not exist");
+        throw std::runtime_error(fmt::format("{} File does not exist: {}",
+                                             LOCATION, fpath.string()));
     }
     if (m_fnc.ext() == ".json") {
         parse_json(fpath);
@@ -143,6 +144,10 @@ std::optional<size_t> RawMasterFile::number_of_rows() const {
     return m_number_of_rows;
 }
 
+std::optional<uint8_t> RawMasterFile::counter_mask() const {
+    return m_counter_mask;
+}
+
 xy RawMasterFile::geometry() const { return m_geometry; }
 
 size_t RawMasterFile::n_modules() const {
@@ -313,15 +318,26 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
         }
 
         // if any of the values are set update the roi
+        // TODO: doesnt it write garbage if one of them is not set
         if (tmp_roi.xmin != 4294967295 || tmp_roi.xmax != 4294967295 ||
             tmp_roi.ymin != 4294967295 || tmp_roi.ymax != 4294967295) {
             tmp_roi.xmax++;
+            // Handle Mythen
+            if (tmp_roi.ymin == -1 && tmp_roi.ymax == -1) {
+                tmp_roi.ymin = 0;
+                tmp_roi.ymax = 0;
+            }
             tmp_roi.ymax++;
             m_roi = tmp_roi;
         }
 
     } catch (const json::out_of_range &e) {
-        LOG(TLogLevel::logERROR) << e.what() << std::endl;
+        // leave the optional empty
+    }
+    try {
+        // TODO: what is the best format to handle
+        m_counter_mask = j.at("Counter Mask");
+    } catch (const json::out_of_range &e) {
         // leave the optional empty
     }