first attempt at gappixels

python/aare/RawFile.py

@@ -4,8 +4,18 @@ import numpy as np
 from .ScanParameters import ScanParameters
 
 class RawFile(_aare.RawFile):
-    def __init__(self, fname, chunk_size = 1):
-        super().__init__(fname)
+    def __init__(self, fname, chunk_size = 1, gap_pixels = False):
+        """Open a raw file for reading.
+
+        Args:
+            fname: Master file (.json or .raw).
+            chunk_size: Frames returned per iteration step.
+            gap_pixels: Insert gap pixels into every frame. True uses the
+                default GapPixels configuration, or pass a GapPixels. Frame
+                shapes then include the gaps. Only Jungfrau and Eiger are
+                supported.
+        """
+        super().__init__(fname, gap_pixels)
         self._chunk_size = chunk_size
 
 

python/aare/__init__.py

@@ -25,6 +25,7 @@ from ._aare import Pedestal_d, Pedestal_f, ClusterFinder_Cluster3x3i, VarCluster
 from ._aare import DetectorType, FrameDiscardPolicy, ReadoutMode
 from ._aare import hitmap
 from ._aare import ROI
+from ._aare import GapPixels, ModuleGaps, insert_gap_pixels, gapped_shape
 from ._aare import xy
 from ._aare import corner 
 

python/src/bind_MultiThreadedFileReader.hpp

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: MPL-2.0
 #pragma once
 
+#include "aare/GapPixels.hpp"
 #include "aare/MultiThreadedFileReader.hpp"
 
 #include <cstddef>
@@ -79,9 +80,21 @@ inline void define_multi_threaded_file_reader_bindings(py::module_ &m) {
     reader.attr("__module__") = "aare.experimental";
     reader
         .def(py::init<std::filesystem::path, size_t, size_t,
-                      std::optional<size_t>>(),
+                      std::optional<size_t>, std::optional<GapPixels>>(),
              py::arg("fname"), py::arg("n_threads"), py::arg("chunk_size"),
              py::arg("total_frames") = py::none(),
+             py::arg("gap_pixels") = py::none())
+        .def(py::init([](std::filesystem::path fname, size_t n_threads,
+                         size_t chunk_size, std::optional<size_t> total_frames,
+                         bool gap_pixels) {
+                 return MultiThreadedFileReader(
+                     std::move(fname), n_threads, chunk_size, total_frames,
+                     gap_pixels ? std::optional<GapPixels>(GapPixels{})
+                                : std::nullopt);
+             }),
+             py::arg("fname"), py::arg("n_threads"), py::arg("chunk_size"),
+             py::arg("total_frames") = py::none(),
+             py::arg("gap_pixels") = false,
              R"doc(
                  Read chunks of detector frames concurrently.
 
@@ -93,7 +106,12 @@ inline void define_multi_threaded_file_reader_bindings(py::module_ &m) {
                      n_threads: Maximum number of worker threads.
                      chunk_size: Number of frames read per claimed chunk.
                      total_frames: Optional frame limit. None reads all frames.
+                     gap_pixels: Insert gap pixels, raw files only. True uses
+                         the default GapPixels configuration. Each worker
+                         uses the configured seed plus its worker index.
                  )doc")
+        .def_property_readonly("gap_pixels",
+                               &MultiThreadedFileReader::gap_pixels)
         .def(
             "read",
             [](MultiThreadedFileReader &self) {

python/src/bind_RawFile.hpp

@@ -3,6 +3,7 @@
 #include "aare/CtbRawFile.hpp"
 #include "aare/File.hpp"
 #include "aare/Frame.hpp"
+#include "aare/GapPixels.hpp"
 #include "aare/RawFile.hpp"
 #include "aare/RawMasterFile.hpp"
 #include "aare/RawSubFile.hpp"
@@ -13,6 +14,7 @@
 #include <cstdint>
 #include <filesystem>
 #include <fmt/format.h>
+#include <memory>
 #include <pybind11/iostream.h>
 #include <pybind11/numpy.h>
 #include <pybind11/pybind11.h>
@@ -34,7 +36,33 @@ std::runtime_error raw_file_read_error(RawFile &file,
 
 void define_raw_file_io_bindings(py::module &m) {
     py::class_<RawFile>(m, "RawFile")
-        .def(py::init<const std::filesystem::path &>())
+        .def(py::init([](const std::filesystem::path &fname,
+                         const GapPixels &gap_pixels) {
+                 return std::make_unique<RawFile>(fname, "r", gap_pixels);
+             }),
+             py::arg("fname"), py::arg("gap_pixels"))
+        .def(py::init([](const std::filesystem::path &fname, bool gap_pixels) {
+                 return std::make_unique<RawFile>(
+                     fname, "r",
+                     gap_pixels ? std::optional<GapPixels>(GapPixels{})
+                                : std::nullopt);
+             }),
+             py::arg("fname"), py::arg("gap_pixels") = false, R"(
+             Open a raw file for reading.
+
+             Parameters
+             ----------
+             fname : path
+                 Master file (.json or .raw).
+             gap_pixels : bool or GapPixels
+                 Insert gap pixels into every frame. True uses the default
+                 GapPixels configuration. Frame shapes, rows() and cols()
+                 then include the gaps. Only Jungfrau and Eiger are
+                 supported.
+             )")
+        .def_property_readonly("gap_pixels", &RawFile::gap_pixels, R"(
+             Gap pixel configuration, or None when reading without gaps.
+             )")
         .def("read_frame",
              [](RawFile &self) {
                  if (self.n_modules_in_roi().size() > 1) {
@@ -118,8 +146,8 @@ void define_raw_file_io_bindings(py::module &m) {
 
                 std::vector<size_t> shape;
                 shape.reserve(2);
-                shape.push_back(self.roi_geometries(roi_index).pixels_y());
-                shape.push_back(self.roi_geometries(roi_index).pixels_x());
+                shape.push_back(self.rows(roi_index));
+                shape.push_back(self.cols(roi_index));
 
                 py::array image =
                     allocate_image_data(self.bytes_per_pixel(), shape);
@@ -177,8 +205,8 @@ void define_raw_file_io_bindings(py::module &m) {
                 for (size_t r = 0; r < number_of_ROIs; r++) {
                     std::vector<size_t> shape;
                     shape.reserve(2);
-                    shape.push_back(self.roi_geometries(r).pixels_y());
-                    shape.push_back(self.roi_geometries(r).pixels_x());
+                    shape.push_back(self.rows(r));
+                    shape.push_back(self.cols(r));
 
                     images[r] =
                         allocate_image_data(self.bytes_per_pixel(), shape);
@@ -229,9 +257,8 @@ void define_raw_file_io_bindings(py::module &m) {
                 if (n_frames == 0) {
                     throw raw_file_read_error(self, "No frames left in file");
                 }
-                std::vector<size_t> shape{
-                    n_frames, self.roi_geometries(roi_index).pixels_y(),
-                    self.roi_geometries(roi_index).pixels_x()};
+                std::vector<size_t> shape{n_frames, self.rows(roi_index),
+                                          self.cols(roi_index)};
 
                 // return headers from all subfiles
                 auto n_mod = self.n_modules_in_roi()[roi_index];

python/src/file.hpp

@@ -3,6 +3,7 @@
 #include "aare/CtbRawFile.hpp"
 #include "aare/File.hpp"
 #include "aare/Frame.hpp"
+#include "aare/GapPixels.hpp"
 #include "aare/ROI.hpp"
 #include "aare/RawFile.hpp"
 #include "aare/RawMasterFile.hpp"
@@ -29,13 +30,37 @@ void define_file_io_bindings(py::module &m) {
                          debug, roundRNumber, detType, version, packetMask);
 
     py::class_<File>(m, "File")
-        .def(py::init([](const std::filesystem::path &fname) {
-            return File(fname, "r", {});
-        }))
-        .def(py::init(
-            [](const std::filesystem::path &fname, const std::string &mode) {
-                return File(fname, mode, {});
-            }))
+        .def(py::init([](const std::filesystem::path &fname,
+                         const std::string &mode,
+                         std::optional<GapPixels> gap_pixels) {
+                 FileConfig cfg;
+                 cfg.gap_pixels = std::move(gap_pixels);
+                 return File(fname, mode, cfg);
+             }),
+             py::arg("fname"), py::arg("mode") = "r",
+             py::arg("gap_pixels") = py::none())
+        .def(py::init([](const std::filesystem::path &fname,
+                         const std::string &mode, bool gap_pixels) {
+                 FileConfig cfg;
+                 if (gap_pixels) {
+                     cfg.gap_pixels = GapPixels{};
+                 }
+                 return File(fname, mode, cfg);
+             }),
+             py::arg("fname"), py::arg("mode") = "r",
+             py::arg("gap_pixels") = false, R"(
+             Open a file for reading.
+
+             Parameters
+             ----------
+             fname : path
+                 Master file (.json, .raw), .npy or .dat file.
+             mode : str
+                 Only "r" is supported.
+             gap_pixels : bool or GapPixels
+                 Insert gap pixels when reading raw files. True uses the
+                 default GapPixels configuration.
+             )")
         .def(py::init<const std::filesystem::path &, const std::string &,
                       const FileConfig &>())
 

python/src/module.cpp

@@ -14,6 +14,7 @@
 #include "bind_Defs.hpp"
 #include "bind_Eta.hpp"
 #include "bind_FastPedestal.hpp"
+#include "bind_GapPixels.hpp"
 #include "bind_Interpolator.hpp"
 #include "bind_MultiThreadedFileReader.hpp"
 #include "bind_Pedestal.hpp"
@@ -156,6 +157,7 @@ PYBIND11_MODULE(_aare, m) {
     register_calculate_3x3eta<int16_t, 3, 3, uint16_t>(m);
 
     define_defs_bindings(m);
+    define_gap_pixels_bindings(m);
 
     using Sum_index_pair_d = Sum_index_pair<double, corner>;
     PYBIND11_NUMPY_DTYPE(Sum_index_pair_d, sum, index);

python/tests/test_MultiThreadedFileReader.py

@@ -3,7 +3,9 @@ import numpy as np
 import pytest
 
 import aare
+from aare import GapPixels, RawFile, insert_gap_pixels
 from aare.experimental import MultiThreadedFileReader
+from test_RawFile import write_module_layout
 
 
 @pytest.fixture
@@ -145,3 +147,38 @@ def test_invalid_configuration(
             chunk_size=chunk_size,
             total_frames=total_frames,
         )
+
+
+def test_gap_pixels_on_raw_files(tmp_path):
+    master_path, expected = write_module_layout(tmp_path, 1, 1, frames=4,
+                                                module_rows=512, module_cols=1024)
+    gaps = GapPixels(fill_value=2)
+    reader = MultiThreadedFileReader(master_path, n_threads=2, chunk_size=1, gap_pixels=gaps)
+    assert reader.gap_pixels == gaps
+    assert (reader.rows, reader.cols) == (514, 1030)
+    frames = reader.read_all()
+    assert frames.shape == (4, 514, 1030)
+    assert (frames == insert_gap_pixels(expected, aare.DetectorType.Jungfrau, gaps)).all()
+
+    reader = MultiThreadedFileReader(master_path, n_threads=2, chunk_size=1, gap_pixels=True)
+    assert reader.gap_pixels == GapPixels()
+    assert reader.read_all().shape == (4, 514, 1030)
+
+    reader = MultiThreadedFileReader(master_path, n_threads=2, chunk_size=1)
+    assert reader.gap_pixels is None
+    assert reader.read_all().shape == (4, 512, 1024)
+
+
+def test_gap_pixels_split_counts_preserve_totals(tmp_path):
+    master_path, expected = write_module_layout(tmp_path, 1, 1, frames=4,
+                                                module_rows=512, module_cols=1024)
+    gaps = GapPixels(split_counts=True, seed=1)
+    frames = MultiThreadedFileReader(master_path, n_threads=2, chunk_size=1, gap_pixels=gaps).read_all()
+    assert frames.shape == (4, 514, 1030)
+    assert (frames.sum(axis=(1, 2)) == expected.sum()).all()
+
+
+def test_gap_pixels_rejected_for_numpy_files(frame_file):
+    path, _ = frame_file
+    with pytest.raises(ValueError):
+        MultiThreadedFileReader(path, n_threads=1, chunk_size=1, gap_pixels=True)

python/tests/test_RawFile.py

@@ -2,6 +2,7 @@
 import pytest
 import json
 from aare import File, RawFile, RawSubFile, DetectorType, ROI, UDPPortPosition, xy
+from aare import GapPixels, ModuleGaps, insert_gap_pixels
 import numpy as np
 
 
@@ -28,6 +29,46 @@ def small_raw_file(tmp_path):
     return master_path
 
 
+MODULE_ROWS = 2
+MODULE_COLS = 3
+
+
+def write_module_layout(directory, layout_rows, layout_cols, frames=2,
+                        module_rows=MODULE_ROWS, module_cols=MODULE_COLS,
+                        detector_type="Jungfrau"):
+    """Write a raw file series with modules in the given layout.
+
+    Modules are numbered as DetectorGeometry orders them, down each column
+    first, and every module holds distinct values so misplacement is visible.
+    Returns the master path and the expected image of frame 0.
+    """
+    master_path = directory / "run_master_0.json"
+    master_path.write_text(json.dumps({
+        "Version": 7.2,
+        "Detector Type": detector_type,
+        "Timing Mode": "auto",
+        "Geometry": {"x": layout_cols, "y": layout_rows},
+        "Image Size in bytes": module_rows * module_cols * 2,
+        "Pixels": {"x": module_cols, "y": module_rows},
+        "Max Frames Per File": 1,
+        "Total Frames": frames,
+        "Frames in File": frames,
+        "Frame Padding": 1,
+        "Frame Discard Policy": "nodiscard",
+    }))
+    expected = np.zeros((module_rows * layout_rows, module_cols * layout_cols), dtype=np.uint16)
+    for module in range(layout_rows * layout_cols):
+        module_row, module_col = module % layout_rows, module // layout_rows
+        pixels = ((np.arange(module_rows * module_cols, dtype=np.uint32) + module * 10 + 1) % 65536).astype(np.uint16)
+        expected[module_row * module_rows:(module_row + 1) * module_rows,
+                 module_col * module_cols:(module_col + 1) * module_cols] = pixels.reshape(module_rows, module_cols)
+        for index in range(frames):
+            data_path = directory / f"run_d{module}_f{index}_0.raw"
+            # 112 byte detector header followed by the module pixels
+            data_path.write_bytes(bytes(112) + pixels.tobytes())
+    return master_path, expected
+
+
 @pytest.mark.parametrize("reader_type", [RawFile, File])
 @pytest.mark.parametrize("legacy_master", [False, True])
 @pytest.mark.parametrize("padding, policy, supported", [
@@ -453,3 +494,109 @@ def test_read_eiger_udp_port_disabled(test_data_path):
         assert len(rois) == 2
         assert rois[0] == ROI(0, 512, 0, 512)
         assert rois[1] == ROI(1024, 1536, 0, 512)
+
+
+@pytest.mark.parametrize("layout", [(1, 1), (2, 1), (1, 2), (2, 2), (3, 2)])
+@pytest.mark.parametrize("reader_type", [RawFile, File])
+def test_raw_file_assembles_module_layout(tmp_path, layout, reader_type):
+    layout_rows, layout_cols = layout
+    master_path, expected = write_module_layout(tmp_path, layout_rows, layout_cols)
+
+    with reader_type(master_path) as reader:
+        # RawFile binds rows/cols as methods (ROI overloads), File as properties
+        rows = reader.rows() if reader_type is RawFile else reader.rows
+        cols = reader.cols() if reader_type is RawFile else reader.cols
+        assert (rows, cols) == expected.shape
+        result = reader.read_frame()
+        image = result[1] if reader_type is RawFile else result
+        assert image.shape == expected.shape
+        assert (image == expected).all()
+
+        reader.seek(0)
+        result = reader.read_n(2)
+        images = result[1] if reader_type is RawFile else result
+        assert images.shape == (2,) + expected.shape
+        assert (images == expected).all()
+
+
+@pytest.mark.parametrize("layout", [(1, 1), (2, 1), (1, 2)])
+def test_raw_file_gap_pixels(tmp_path, layout):
+    layout_rows, layout_cols = layout
+    master_path, expected = write_module_layout(tmp_path, layout_rows, layout_cols,
+                                                module_rows=512, module_cols=1024)
+    gaps = GapPixels(fill_value=5)
+    with RawFile(master_path, gap_pixels=gaps) as reader:
+        assert reader.gap_pixels == gaps
+        assert (reader.rows(), reader.cols()) == (512 * layout_rows + 2 * (layout_rows * 2 - 1),
+                                                  1024 * layout_cols + 2 * (layout_cols * 4 - 1))
+        header, image = reader.read_frame()
+        assert header.size == layout_rows * layout_cols
+        assert (image == insert_gap_pixels(expected, DetectorType.Jungfrau, gaps)).all()
+        reader.seek(0)
+        _, images = reader.read_n(2)
+        assert images.shape == (2, reader.rows(), reader.cols())
+        assert (images[1] == image).all()
+
+    with RawFile(master_path) as reader:
+        assert reader.gap_pixels is None
+        assert (reader.rows(), reader.cols()) == expected.shape
+
+    with RawFile(master_path, gap_pixels=True) as reader:
+        assert reader.gap_pixels == GapPixels()
+        _, image = reader.read_frame()
+        assert (image == insert_gap_pixels(expected, DetectorType.Jungfrau)).all()
+
+    with File(master_path, gap_pixels=True) as reader:
+        assert (reader.rows, reader.cols) == insert_gap_pixels(expected, DetectorType.Jungfrau).shape
+        assert (reader.read_frame() == insert_gap_pixels(expected, DetectorType.Jungfrau)).all()
+
+    module_gaps = GapPixels(module_gaps=ModuleGaps(8, 36))
+    with File(master_path, gap_pixels=module_gaps) as reader:
+        assert (reader.read_frame() == insert_gap_pixels(expected, DetectorType.Jungfrau, module_gaps)).all()
+
+
+def test_raw_file_gap_pixels_split_counts(tmp_path):
+    master_path, expected = write_module_layout(tmp_path, 1, 1, module_rows=512, module_cols=1024)
+    gaps = GapPixels(split_counts=True, seed=11)
+    with RawFile(master_path, gap_pixels=gaps) as reader:
+        _, image = reader.read_frame()
+    assert (image == insert_gap_pixels(expected, DetectorType.Jungfrau, gaps)).all()
+    assert image.sum() == expected.sum()
+
+
+def test_raw_file_gap_pixels_unsupported_detector(tmp_path):
+    master_path, _ = write_module_layout(tmp_path, 1, 1, detector_type="Moench")
+    RawFile(master_path)
+    with pytest.raises(ValueError):
+        RawFile(master_path, gap_pixels=True)
+    with pytest.raises(ValueError):
+        File(master_path, gap_pixels=True)
+
+
+@pytest.mark.withdata
+def test_raw_file_gap_pixels_recorded_jungfrau(test_data_path):
+    path = test_data_path / "raw/jungfrau/jungfrau_single_master_0.json"
+    gaps = GapPixels(fill_value=3)
+    with RawFile(path, gap_pixels=gaps) as gapped, RawFile(path) as plain:
+        assert (gapped.rows(), gapped.cols()) == (514, 1030)
+        _, image = gapped.read_frame()
+        _, reference = plain.read_frame()
+        assert image.shape == (514, 1030)
+        assert (image == insert_gap_pixels(reference, DetectorType.Jungfrau, gaps)).all()
+
+
+@pytest.mark.withdata
+def test_raw_file_gap_pixels_recorded_rois(test_data_path):
+    path = test_data_path / "raw/ROITestData/MultipleROIs/run_master_0.json"
+    with RawFile(path, gap_pixels=True) as gapped, RawFile(path) as plain:
+        assert gapped.num_rois == 2
+        _, images = gapped.read_rois()
+        _, references = plain.read_rois()
+        rois = plain.master.rois
+        assert images[0].shape == (303, 103)
+        assert images[1].shape == (101, 103)
+        for image, reference, roi in zip(images, references, rois):
+            assert (image == insert_gap_pixels(reference, DetectorType.Jungfrau, roi=roi)).all()
+        gapped.seek(0)
+        _, frames = gapped.read_n_with_roi(2, roi_index=1)
+        assert frames.shape == (2, 101, 103)