Read raw files with gap pixels

RawFile takes an optional GapPixels configuration and assembles every
frame with the gaps inserted: the gap lines are filled, each module part
is scattered through the scratch buffer into its gapped position and,
if configured, the counts of the chip edge pixels are split. rows(),
cols(), pixels_per_frame() and bytes_per_frame() include the gaps, for
the full frame and for each ROI. File forwards FileConfig::gap_pixels to
RawFile and rejects it for other file types. Only Jungfrau and Eiger are
supported.

Python RawFile and File accept gap_pixels as a bool or a GapPixels and
expose it as a property. Frame shapes come from rows() and cols().

Adds the module layout assembly tests and the raw file benchmark that
the assembly refactor went in without, since the gap tests build on the
generalized raw file helper.

python/aare/RawFile.py

@@ -4,8 +4,18 @@ import numpy as np
 from .ScanParameters import ScanParameters
 
 class RawFile(_aare.RawFile):
-    def __init__(self, fname, chunk_size = 1):
-        super().__init__(fname)
+    def __init__(self, fname, chunk_size = 1, gap_pixels = False):
+        """Open a raw file for reading.
+
+        Args:
+            fname: Master file (.json or .raw).
+            chunk_size: Frames returned per iteration step.
+            gap_pixels: Insert gap pixels into every frame. True uses the
+                default GapPixels configuration, or pass a GapPixels. Frame
+                shapes then include the gaps. Only Jungfrau and Eiger are
+                supported.
+        """
+        super().__init__(fname, gap_pixels)
         self._chunk_size = chunk_size
 
 

python/src/bind_RawFile.hpp

@@ -3,6 +3,7 @@
 #include "aare/CtbRawFile.hpp"
 #include "aare/File.hpp"
 #include "aare/Frame.hpp"
+#include "aare/GapPixels.hpp"
 #include "aare/RawFile.hpp"
 #include "aare/RawMasterFile.hpp"
 #include "aare/RawSubFile.hpp"
@@ -13,6 +14,7 @@
 #include <cstdint>
 #include <filesystem>
 #include <fmt/format.h>
+#include <memory>
 #include <pybind11/iostream.h>
 #include <pybind11/numpy.h>
 #include <pybind11/pybind11.h>
@@ -34,7 +36,33 @@ std::runtime_error raw_file_read_error(RawFile &file,
 
 void define_raw_file_io_bindings(py::module &m) {
     py::class_<RawFile>(m, "RawFile")
-        .def(py::init<const std::filesystem::path &>())
+        .def(py::init([](const std::filesystem::path &fname,
+                         const GapPixels &gap_pixels) {
+                 return std::make_unique<RawFile>(fname, "r", gap_pixels);
+             }),
+             py::arg("fname"), py::arg("gap_pixels"))
+        .def(py::init([](const std::filesystem::path &fname, bool gap_pixels) {
+                 return std::make_unique<RawFile>(
+                     fname, "r",
+                     gap_pixels ? std::optional<GapPixels>(GapPixels{})
+                                : std::nullopt);
+             }),
+             py::arg("fname"), py::arg("gap_pixels") = false, R"(
+             Open a raw file for reading.
+
+             Parameters
+             ----------
+             fname : path
+                 Master file (.json or .raw).
+             gap_pixels : bool or GapPixels
+                 Insert gap pixels into every frame. True uses the default
+                 GapPixels configuration. Frame shapes, rows() and cols()
+                 then include the gaps. Only Jungfrau and Eiger are
+                 supported.
+             )")
+        .def_property_readonly("gap_pixels", &RawFile::gap_pixels, R"(
+             Gap pixel configuration, or None when reading without gaps.
+             )")
         .def("read_frame",
              [](RawFile &self) {
                  if (self.n_modules_in_roi().size() > 1) {
@@ -118,8 +146,8 @@ void define_raw_file_io_bindings(py::module &m) {
 
                 std::vector<size_t> shape;
                 shape.reserve(2);
-                shape.push_back(self.roi_geometries(roi_index).pixels_y());
-                shape.push_back(self.roi_geometries(roi_index).pixels_x());
+                shape.push_back(self.rows(roi_index));
+                shape.push_back(self.cols(roi_index));
 
                 py::array image =
                     allocate_image_data(self.bytes_per_pixel(), shape);
@@ -177,8 +205,8 @@ void define_raw_file_io_bindings(py::module &m) {
                 for (size_t r = 0; r < number_of_ROIs; r++) {
                     std::vector<size_t> shape;
                     shape.reserve(2);
-                    shape.push_back(self.roi_geometries(r).pixels_y());
-                    shape.push_back(self.roi_geometries(r).pixels_x());
+                    shape.push_back(self.rows(r));
+                    shape.push_back(self.cols(r));
 
                     images[r] =
                         allocate_image_data(self.bytes_per_pixel(), shape);
@@ -229,9 +257,8 @@ void define_raw_file_io_bindings(py::module &m) {
                 if (n_frames == 0) {
                     throw raw_file_read_error(self, "No frames left in file");
                 }
-                std::vector<size_t> shape{
-                    n_frames, self.roi_geometries(roi_index).pixels_y(),
-                    self.roi_geometries(roi_index).pixels_x()};
+                std::vector<size_t> shape{n_frames, self.rows(roi_index),
+                                          self.cols(roi_index)};
 
                 // return headers from all subfiles
                 auto n_mod = self.n_modules_in_roi()[roi_index];

python/src/file.hpp

@@ -3,6 +3,7 @@
 #include "aare/CtbRawFile.hpp"
 #include "aare/File.hpp"
 #include "aare/Frame.hpp"
+#include "aare/GapPixels.hpp"
 #include "aare/ROI.hpp"
 #include "aare/RawFile.hpp"
 #include "aare/RawMasterFile.hpp"
@@ -29,13 +30,37 @@ void define_file_io_bindings(py::module &m) {
                          debug, roundRNumber, detType, version, packetMask);
 
     py::class_<File>(m, "File")
-        .def(py::init([](const std::filesystem::path &fname) {
-            return File(fname, "r", {});
-        }))
-        .def(py::init(
-            [](const std::filesystem::path &fname, const std::string &mode) {
-                return File(fname, mode, {});
-            }))
+        .def(py::init([](const std::filesystem::path &fname,
+                         const std::string &mode,
+                         std::optional<GapPixels> gap_pixels) {
+                 FileConfig cfg;
+                 cfg.gap_pixels = std::move(gap_pixels);
+                 return File(fname, mode, cfg);
+             }),
+             py::arg("fname"), py::arg("mode") = "r",
+             py::arg("gap_pixels") = py::none())
+        .def(py::init([](const std::filesystem::path &fname,
+                         const std::string &mode, bool gap_pixels) {
+                 FileConfig cfg;
+                 if (gap_pixels) {
+                     cfg.gap_pixels = GapPixels{};
+                 }
+                 return File(fname, mode, cfg);
+             }),
+             py::arg("fname"), py::arg("mode") = "r",
+             py::arg("gap_pixels") = false, R"(
+             Open a file for reading.
+
+             Parameters
+             ----------
+             fname : path
+                 Master file (.json, .raw), .npy or .dat file.
+             mode : str
+                 Only "r" is supported.
+             gap_pixels : bool or GapPixels
+                 Insert gap pixels when reading raw files. True uses the
+                 default GapPixels configuration.
+             )")
         .def(py::init<const std::filesystem::path &, const std::string &,
                       const FileConfig &>())
 

python/tests/test_RawFile.py

@@ -2,6 +2,7 @@
 import pytest
 import json
 from aare import File, RawFile, RawSubFile, DetectorType, ROI, UDPPortPosition, xy
+from aare import GapPixels, ModuleGaps, insert_gap_pixels
 import numpy as np
 
 
@@ -28,6 +29,46 @@ def small_raw_file(tmp_path):
     return master_path
 
 
+MODULE_ROWS = 2
+MODULE_COLS = 3
+
+
+def write_module_layout(directory, layout_rows, layout_cols, frames=2,
+                        module_rows=MODULE_ROWS, module_cols=MODULE_COLS,
+                        detector_type="Jungfrau"):
+    """Write a raw file series with modules in the given layout.
+
+    Modules are numbered as DetectorGeometry orders them, down each column
+    first, and every module holds distinct values so misplacement is visible.
+    Returns the master path and the expected image of frame 0.
+    """
+    master_path = directory / "run_master_0.json"
+    master_path.write_text(json.dumps({
+        "Version": 7.2,
+        "Detector Type": detector_type,
+        "Timing Mode": "auto",
+        "Geometry": {"x": layout_cols, "y": layout_rows},
+        "Image Size in bytes": module_rows * module_cols * 2,
+        "Pixels": {"x": module_cols, "y": module_rows},
+        "Max Frames Per File": 1,
+        "Total Frames": frames,
+        "Frames in File": frames,
+        "Frame Padding": 1,
+        "Frame Discard Policy": "nodiscard",
+    }))
+    expected = np.zeros((module_rows * layout_rows, module_cols * layout_cols), dtype=np.uint16)
+    for module in range(layout_rows * layout_cols):
+        module_row, module_col = module % layout_rows, module // layout_rows
+        pixels = ((np.arange(module_rows * module_cols, dtype=np.uint32) + module * 10 + 1) % 65536).astype(np.uint16)
+        expected[module_row * module_rows:(module_row + 1) * module_rows,
+                 module_col * module_cols:(module_col + 1) * module_cols] = pixels.reshape(module_rows, module_cols)
+        for index in range(frames):
+            data_path = directory / f"run_d{module}_f{index}_0.raw"
+            # 112 byte detector header followed by the module pixels
+            data_path.write_bytes(bytes(112) + pixels.tobytes())
+    return master_path, expected
+
+
 @pytest.mark.parametrize("reader_type", [RawFile, File])
 @pytest.mark.parametrize("legacy_master", [False, True])
 @pytest.mark.parametrize("padding, policy, supported", [
@@ -453,3 +494,110 @@ def test_read_eiger_udp_port_disabled(test_data_path):
         assert len(rois) == 2
         assert rois[0] == ROI(0, 512, 0, 512)
         assert rois[1] == ROI(1024, 1536, 0, 512)
+
+
+@pytest.mark.parametrize("layout", [(1, 1), (2, 1), (1, 2), (2, 2), (3, 2)])
+@pytest.mark.parametrize("reader_type", [RawFile, File])
+def test_raw_file_assembles_module_layout(tmp_path, layout, reader_type):
+    layout_rows, layout_cols = layout
+    master_path, expected = write_module_layout(tmp_path, layout_rows, layout_cols)
+
+    with reader_type(master_path) as reader:
+        # RawFile binds rows/cols as methods (ROI overloads), File as properties
+        rows = reader.rows() if reader_type is RawFile else reader.rows
+        cols = reader.cols() if reader_type is RawFile else reader.cols
+        assert (rows, cols) == expected.shape
+        result = reader.read_frame()
+        image = result[1] if reader_type is RawFile else result
+        assert image.shape == expected.shape
+        assert (image == expected).all()
+
+        reader.seek(0)
+        result = reader.read_n(2)
+        images = result[1] if reader_type is RawFile else result
+        assert images.shape == (2,) + expected.shape
+        assert (images == expected).all()
+
+
+@pytest.mark.parametrize("layout", [(1, 1), (2, 1), (1, 2)])
+def test_raw_file_gap_pixels(tmp_path, layout):
+    layout_rows, layout_cols = layout
+    master_path, expected = write_module_layout(tmp_path, layout_rows, layout_cols,
+                                                module_rows=512, module_cols=1024)
+    gaps = GapPixels(fill_value=5)
+    with RawFile(master_path, gap_pixels=gaps) as reader:
+        assert reader.gap_pixels == gaps
+        # one chip boundary per module row, three per module column and no
+        # gap between modules
+        assert (reader.rows(), reader.cols()) == ((512 + 2) * layout_rows, (1024 + 6) * layout_cols)
+        header, image = reader.read_frame()
+        assert header.size == layout_rows * layout_cols
+        assert (image == insert_gap_pixels(expected, gaps)).all()
+        reader.seek(0)
+        _, images = reader.read_n(2)
+        assert images.shape == (2, reader.rows(), reader.cols())
+        assert (images[1] == image).all()
+
+    with RawFile(master_path) as reader:
+        assert reader.gap_pixels is None
+        assert (reader.rows(), reader.cols()) == expected.shape
+
+    with RawFile(master_path, gap_pixels=True) as reader:
+        assert reader.gap_pixels == GapPixels()
+        _, image = reader.read_frame()
+        assert (image == insert_gap_pixels(expected)).all()
+
+    with File(master_path, gap_pixels=True) as reader:
+        assert (reader.rows, reader.cols) == insert_gap_pixels(expected).shape
+        assert (reader.read_frame() == insert_gap_pixels(expected)).all()
+
+    module_gaps = GapPixels(module_gaps=ModuleGaps(8, 36))
+    with File(master_path, gap_pixels=module_gaps) as reader:
+        assert (reader.read_frame() == insert_gap_pixels(expected, module_gaps)).all()
+
+
+def test_raw_file_gap_pixels_split_counts(tmp_path):
+    master_path, expected = write_module_layout(tmp_path, 1, 1, module_rows=512, module_cols=1024)
+    gaps = GapPixels(split_counts=True, seed=11)
+    with RawFile(master_path, gap_pixels=gaps) as reader:
+        _, image = reader.read_frame()
+    assert (image == insert_gap_pixels(expected, gaps)).all()
+    assert image.sum() == expected.sum()
+
+
+def test_raw_file_gap_pixels_unsupported_detector(tmp_path):
+    master_path, _ = write_module_layout(tmp_path, 1, 1, detector_type="Moench")
+    RawFile(master_path)
+    with pytest.raises(ValueError):
+        RawFile(master_path, gap_pixels=True)
+    with pytest.raises(ValueError):
+        File(master_path, gap_pixels=True)
+
+
+@pytest.mark.withdata
+def test_raw_file_gap_pixels_recorded_jungfrau(test_data_path):
+    path = test_data_path / "raw/jungfrau/jungfrau_single_master_0.json"
+    gaps = GapPixels(fill_value=3)
+    with RawFile(path, gap_pixels=gaps) as gapped, RawFile(path) as plain:
+        assert (gapped.rows(), gapped.cols()) == (514, 1030)
+        _, image = gapped.read_frame()
+        _, reference = plain.read_frame()
+        assert image.shape == (514, 1030)
+        assert (image == insert_gap_pixels(reference, gaps)).all()
+
+
+@pytest.mark.withdata
+def test_raw_file_gap_pixels_recorded_rois(test_data_path):
+    path = test_data_path / "raw/ROITestData/MultipleROIs/run_master_0.json"
+    with RawFile(path, gap_pixels=True) as gapped, RawFile(path) as plain:
+        assert gapped.num_rois == 2
+        _, images = gapped.read_rois()
+        _, references = plain.read_rois()
+        rois = plain.master.rois
+        assert images[0].shape == (303, 103)
+        assert images[1].shape == (101, 103)
+        for image, reference, roi in zip(images, references, rois):
+            assert (image == insert_gap_pixels(reference, roi=roi)).all()
+        gapped.seek(0)
+        _, frames = gapped.read_n_with_roi(2, roi_index=1)
+        assert frames.shape == (2, 101, 103)