Merge branch 'main' into dev/strixels/remap_simple

python/aare/RawFile.py

@@ -4,16 +4,18 @@ import numpy as np
 from .ScanParameters import ScanParameters
 
 class RawFile(_aare.RawFile):
-    """
-    Class to read Raw files produced by slsDetectorPackage 
+    def __init__(self, fname, chunk_size = 1, gap_pixels = False):
+        """Open a raw file for reading.
 
-    Parameters:
-        fname (str): Path to the master file.
-        chunk_size (int, optional): Number of frames to read at a time. Defaults to 1.
-        strixeltransform (list, optional): List of strixel transform objects. Defaults to None.
-    """
-    def __init__(self, fname, chunk_size = 1, strixeltransform : list = None):
-        super().__init__(fname)
+        Args:
+            fname: Master file (.json or .raw).
+            chunk_size: Frames returned per iteration step.
+            gap_pixels: Insert gap pixels into every frame. True uses the
+                default GapPixels configuration, or pass a GapPixels. Frame
+                shapes then include the gaps. Only Jungfrau and Eiger are
+                supported.
+        """
+        super().__init__(fname, gap_pixels)
         self._chunk_size = chunk_size
         self._strixeltransform : list = strixeltransform
         if self._strixeltransform is not None:

python/src/bind_RawFile.hpp

@@ -3,6 +3,7 @@
 #include "aare/CtbRawFile.hpp"
 #include "aare/File.hpp"
 #include "aare/Frame.hpp"
+#include "aare/GapPixels.hpp"
 #include "aare/RawFile.hpp"
 #include "aare/RawMasterFile.hpp"
 #include "aare/RawSubFile.hpp"
@@ -13,6 +14,7 @@
 #include <cstdint>
 #include <filesystem>
 #include <fmt/format.h>
+#include <memory>
 #include <pybind11/iostream.h>
 #include <pybind11/numpy.h>
 #include <pybind11/pybind11.h>
@@ -34,7 +36,33 @@ std::runtime_error raw_file_read_error(RawFile &file,
 
 void define_raw_file_io_bindings(py::module &m) {
     py::class_<RawFile>(m, "RawFile")
-        .def(py::init<const std::filesystem::path &>())
+        .def(py::init([](const std::filesystem::path &fname,
+                         const GapPixels &gap_pixels) {
+                 return std::make_unique<RawFile>(fname, "r", gap_pixels);
+             }),
+             py::arg("fname"), py::arg("gap_pixels"))
+        .def(py::init([](const std::filesystem::path &fname, bool gap_pixels) {
+                 return std::make_unique<RawFile>(
+                     fname, "r",
+                     gap_pixels ? std::optional<GapPixels>(GapPixels{})
+                                : std::nullopt);
+             }),
+             py::arg("fname"), py::arg("gap_pixels") = false, R"(
+             Open a raw file for reading.
+
+             Parameters
+             ----------
+             fname : path
+                 Master file (.json or .raw).
+             gap_pixels : bool or GapPixels
+                 Insert gap pixels into every frame. True uses the default
+                 GapPixels configuration. Frame shapes, rows() and cols()
+                 then include the gaps. Only Jungfrau and Eiger are
+                 supported.
+             )")
+        .def_property_readonly("gap_pixels", &RawFile::gap_pixels, R"(
+             Gap pixel configuration, or None when reading without gaps.
+             )")
         .def("read_frame",
              [](RawFile &self) {
                  if (self.n_modules_in_roi().size() > 1) {
@@ -118,8 +146,8 @@ void define_raw_file_io_bindings(py::module &m) {
 
                 std::vector<size_t> shape;
                 shape.reserve(2);
-                shape.push_back(self.roi_geometries(roi_index).pixels_y());
-                shape.push_back(self.roi_geometries(roi_index).pixels_x());
+                shape.push_back(self.rows(roi_index));
+                shape.push_back(self.cols(roi_index));
 
                 py::array image =
                     allocate_image_data(self.bytes_per_pixel(), shape);
@@ -177,8 +205,8 @@ void define_raw_file_io_bindings(py::module &m) {
                 for (size_t r = 0; r < number_of_ROIs; r++) {
                     std::vector<size_t> shape;
                     shape.reserve(2);
-                    shape.push_back(self.roi_geometries(r).pixels_y());
-                    shape.push_back(self.roi_geometries(r).pixels_x());
+                    shape.push_back(self.rows(r));
+                    shape.push_back(self.cols(r));
 
                     images[r] =
                         allocate_image_data(self.bytes_per_pixel(), shape);
@@ -229,9 +257,8 @@ void define_raw_file_io_bindings(py::module &m) {
                 if (n_frames == 0) {
                     throw raw_file_read_error(self, "No frames left in file");
                 }
-                std::vector<size_t> shape{
-                    n_frames, self.roi_geometries(roi_index).pixels_y(),
-                    self.roi_geometries(roi_index).pixels_x()};
+                std::vector<size_t> shape{n_frames, self.rows(roi_index),
+                                          self.cols(roi_index)};
 
                 // return headers from all subfiles
                 auto n_mod = self.n_modules_in_roi()[roi_index];

python/tests/test_RawFile.py

@@ -2,6 +2,7 @@
 import pytest
 import json
 from aare import File, RawFile, RawSubFile, DetectorType, ROI, UDPPortPosition, xy
+from aare import GapPixels, ModuleGaps, insert_gap_pixels
 import numpy as np
 
 from aare import strixelremap
@@ -31,6 +32,46 @@ def small_raw_file(tmp_path):
     return master_path
 
 
+MODULE_ROWS = 2
+MODULE_COLS = 3
+
+
+def write_module_layout(directory, layout_rows, layout_cols, frames=2,
+                        module_rows=MODULE_ROWS, module_cols=MODULE_COLS,
+                        detector_type="Jungfrau"):
+    """Write a raw file series with modules in the given layout.
+
+    Modules are numbered as DetectorGeometry orders them, down each column
+    first, and every module holds distinct values so misplacement is visible.
+    Returns the master path and the expected image of frame 0.
+    """
+    master_path = directory / "run_master_0.json"
+    master_path.write_text(json.dumps({
+        "Version": 7.2,
+        "Detector Type": detector_type,
+        "Timing Mode": "auto",
+        "Geometry": {"x": layout_cols, "y": layout_rows},
+        "Image Size in bytes": module_rows * module_cols * 2,
+        "Pixels": {"x": module_cols, "y": module_rows},
+        "Max Frames Per File": 1,
+        "Total Frames": frames,
+        "Frames in File": frames,
+        "Frame Padding": 1,
+        "Frame Discard Policy": "nodiscard",
+    }))
+    expected = np.zeros((module_rows * layout_rows, module_cols * layout_cols), dtype=np.uint16)
+    for module in range(layout_rows * layout_cols):
+        module_row, module_col = module % layout_rows, module // layout_rows
+        pixels = ((np.arange(module_rows * module_cols, dtype=np.uint32) + module * 10 + 1) % 65536).astype(np.uint16)
+        expected[module_row * module_rows:(module_row + 1) * module_rows,
+                 module_col * module_cols:(module_col + 1) * module_cols] = pixels.reshape(module_rows, module_cols)
+        for index in range(frames):
+            data_path = directory / f"run_d{module}_f{index}_0.raw"
+            # 112 byte detector header followed by the module pixels
+            data_path.write_bytes(bytes(112) + pixels.tobytes())
+    return master_path, expected
+
+
 @pytest.mark.parametrize("reader_type", [RawFile, File])
 @pytest.mark.parametrize("legacy_master", [False, True])
 @pytest.mark.parametrize("padding, policy, supported", [
@@ -469,4 +510,120 @@ def test_RawFile_with_strixeltransform():
         assert len(frames[0]) == 3
         assert frames[0][0].shape == (165, 79)
         assert frames[0][1].shape == (320, 47)
-        assert frames[0][2].shape == (476, 59)
+        assert frames[0][2].shape == (476, 59)
+
+@pytest.mark.parametrize("layout", [(1, 1), (2, 1), (1, 2), (2, 2), (3, 2)])
+@pytest.mark.parametrize("reader_type", [RawFile, File])
+def test_raw_file_assembles_module_layout(tmp_path, layout, reader_type):
+    layout_rows, layout_cols = layout
+    master_path, expected = write_module_layout(tmp_path, layout_rows, layout_cols)
+
+    with reader_type(master_path) as reader:
+        # RawFile binds rows/cols as methods (ROI overloads), File as properties
+        rows = reader.rows() if reader_type is RawFile else reader.rows
+        cols = reader.cols() if reader_type is RawFile else reader.cols
+        assert (rows, cols) == expected.shape
+        result = reader.read_frame()
+        image = result[1] if reader_type is RawFile else result
+        assert image.shape == expected.shape
+        assert (image == expected).all()
+
+        reader.seek(0)
+        result = reader.read_n(2)
+        images = result[1] if reader_type is RawFile else result
+        assert images.shape == (2,) + expected.shape
+        assert (images == expected).all()
+
+
+@pytest.mark.parametrize("layout", [(1, 1), (2, 1), (1, 2)])
+def test_raw_file_gap_pixels(tmp_path, layout):
+    layout_rows, layout_cols = layout
+    master_path, expected = write_module_layout(tmp_path, layout_rows, layout_cols,
+                                                module_rows=512, module_cols=1024)
+    gaps = GapPixels(fill_value=5)
+    with RawFile(master_path, gap_pixels=gaps) as reader:
+        assert reader.gap_pixels == gaps
+        # one chip boundary per module row, three per module column and no
+        # gap between modules
+        assert (reader.rows(), reader.cols()) == ((512 + 2) * layout_rows, (1024 + 6) * layout_cols)
+        header, image = reader.read_frame()
+        assert header.size == layout_rows * layout_cols
+        assert (image == insert_gap_pixels(expected, gaps)).all()
+        reader.seek(0)
+        _, images = reader.read_n(2)
+        assert images.shape == (2, reader.rows(), reader.cols())
+        assert (images[1] == image).all()
+
+    with RawFile(master_path) as reader:
+        assert reader.gap_pixels is None
+        assert (reader.rows(), reader.cols()) == expected.shape
+
+    with RawFile(master_path, gap_pixels=True) as reader:
+        assert reader.gap_pixels == GapPixels()
+        assert (reader.rows(), reader.cols()) == insert_gap_pixels(expected).shape
+        _, image = reader.read_frame()
+        assert (image == insert_gap_pixels(expected)).all()
+
+    module_gaps = GapPixels(module_gaps=ModuleGaps(8, 36))
+    with RawFile(master_path, gap_pixels=module_gaps) as reader:
+        assert (reader.rows(), reader.cols()) == insert_gap_pixels(expected, module_gaps).shape
+        _, image = reader.read_frame()
+        assert (image == insert_gap_pixels(expected, module_gaps)).all()
+
+
+def test_raw_file_gap_pixels_split_counts(tmp_path):
+    # one Eiger module: four 256x512 ports in a 2x2 layout. Splitting is
+    # rejected on raw Jungfrau data. The plain read is the reference since
+    # Eiger ports are row flipped on reading.
+    master_path, _ = write_module_layout(tmp_path, 2, 2, module_rows=256, module_cols=512,
+                                         detector_type="Eiger")
+    gaps = GapPixels(split_counts=True, seed=11)
+    with RawFile(master_path) as reader:
+        _, plain = reader.read_frame()
+    with RawFile(master_path, gap_pixels=gaps) as reader:
+        _, image = reader.read_frame()
+    assert (image == insert_gap_pixels(plain, gaps)).all()
+    assert image.sum() == plain.sum()
+
+
+def test_raw_file_gap_pixels_split_counts_rejected_for_jungfrau(tmp_path):
+    master_path, _ = write_module_layout(tmp_path, 1, 1, module_rows=512, module_cols=1024)
+    RawFile(master_path, gap_pixels=GapPixels())
+    with pytest.raises(ValueError):
+        RawFile(master_path, gap_pixels=GapPixels(split_counts=True))
+
+
+def test_raw_file_gap_pixels_unsupported_detector(tmp_path):
+    master_path, _ = write_module_layout(tmp_path, 1, 1, detector_type="Moench")
+    RawFile(master_path)
+    with pytest.raises(ValueError):
+        RawFile(master_path, gap_pixels=True)
+
+
+@pytest.mark.withdata
+def test_raw_file_gap_pixels_recorded_jungfrau(test_data_path):
+    path = test_data_path / "raw/jungfrau/jungfrau_single_master_0.json"
+    gaps = GapPixels(fill_value=3)
+    with RawFile(path, gap_pixels=gaps) as gapped, RawFile(path) as plain:
+        assert (gapped.rows(), gapped.cols()) == (514, 1030)
+        _, image = gapped.read_frame()
+        _, reference = plain.read_frame()
+        assert image.shape == (514, 1030)
+        assert (image == insert_gap_pixels(reference, gaps)).all()
+
+
+@pytest.mark.withdata
+def test_raw_file_gap_pixels_recorded_rois(test_data_path):
+    path = test_data_path / "raw/ROITestData/MultipleROIs/run_master_0.json"
+    with RawFile(path, gap_pixels=True) as gapped, RawFile(path) as plain:
+        assert gapped.num_rois == 2
+        _, images = gapped.read_rois()
+        _, references = plain.read_rois()
+        rois = plain.master.rois
+        assert images[0].shape == (303, 103)
+        assert images[1].shape == (101, 103)
+        for image, reference, roi in zip(images, references, rois):
+            assert (image == insert_gap_pixels(reference, roi=roi)).all()
+        gapped.seek(0)
+        _, frames = gapped.read_n_with_roi(2, roi_index=1)
+        assert frames.shape == (2, 101, 103)