better error and update frames (#365)
Build on RHEL9 / build (push) Successful in 2m34s
Build on RHEL8 / build (push) Successful in 3m13s
Run tests using data on local RHEL8 / build (push) Failing after 4m14s
Build on local RHEL8 / build (push) Successful in 2m54s

- Updated error message to mention out of range and which file it
applies to
- If there are different number of frames in the subfiles, take the
minimum and print a warning

Closes: #363 
Closes: #362
This commit is contained in:
Erik Fröjdh authored and GitHub committed 2026-09-09 17:07:08 +02:00
1 parent afce91b358
commit 3431bbe117
15 files changed
+787 -107

No files matched your search

+1 -1
View File
@@ -43,7 +43,7 @@ class RawFile(_aare.RawFile):
Returns:
int: Number of frames in file.
"""
return super().frames_in_file
return self.total_frames
def __enter__(self):
return self
+33 -15
View File
@@ -11,6 +11,7 @@
#include <cstdint>
#include <filesystem>
#include <fmt/format.h>
#include <pybind11/iostream.h>
#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>
@@ -21,14 +22,23 @@
namespace py = pybind11;
using namespace ::aare;
namespace {
std::runtime_error raw_file_read_error(RawFile &file,
const std::string &message) {
return std::runtime_error(fmt::format(
"Error reading frame index {} from file '{}': {}", file.tell(),
file.master().master_fname().string(), message));
}
} // namespace
void define_raw_file_io_bindings(py::module &m) {
py::class_<RawFile>(m, "RawFile")
.def(py::init<const std::filesystem::path &>())
.def("read_frame",
[](RawFile &self) {
if (self.n_modules_in_roi().size() > 1) {
throw std::runtime_error(
"File contains multiple ROIs - use read_ROIs()");
throw raw_file_read_error(
self, "File contains multiple ROIs - use read_ROIs()");
}
std::vector<size_t> shape;
@@ -51,16 +61,19 @@ void define_raw_file_io_bindings(py::module &m) {
"read_n",
[](RawFile &self, size_t n_frames) {
if (self.n_modules_in_roi().size() > 1) {
throw std::runtime_error(
throw raw_file_read_error(
self,
"File contains multiple ROIs - use read_n_ROIs() to "
"read a specific ROI or use read_ROIs and "
"read one frame at a time.");
}
// adjust for actual frames left in the file
n_frames =
std::min(n_frames, self.total_frames() - self.tell());
std::min(n_frames, self.tell() < self.total_frames()
? self.total_frames() - self.tell()
: size_t{0});
if (n_frames == 0) {
throw std::runtime_error("No frames left in file");
throw raw_file_read_error(self, "No frames left in file");
}
std::vector<size_t> shape{n_frames, self.rows(), self.cols()};
@@ -89,12 +102,13 @@ void define_raw_file_io_bindings(py::module &m) {
"read_roi",
[](RawFile &self, const size_t roi_index) {
if (self.num_rois() == 0) {
throw std::runtime_error(LOCATION + "No ROIs defined.");
throw raw_file_read_error(self,
LOCATION + "No ROIs defined.");
}
if (roi_index >= self.num_rois()) {
throw std::runtime_error(LOCATION +
"ROI index out of range.");
throw raw_file_read_error(
self, LOCATION + "ROI index out of range.");
}
// return headers from all subfiles
@@ -141,7 +155,8 @@ void define_raw_file_io_bindings(py::module &m) {
"read_rois",
[](RawFile &self) {
if (self.num_rois() == 0) {
throw std::runtime_error(LOCATION + "No ROIs defined.");
throw raw_file_read_error(self,
LOCATION + "No ROIs defined.");
}
size_t number_of_ROIs = self.num_rois();
@@ -196,19 +211,22 @@ void define_raw_file_io_bindings(py::module &m) {
"read_n_with_roi",
[](RawFile &self, const size_t num_frames, const size_t roi_index) {
if (self.num_rois() == 0) {
throw std::runtime_error(LOCATION + "No ROIs defined.");
throw raw_file_read_error(self,
LOCATION + "No ROIs defined.");
}
if (roi_index >= self.num_rois()) {
throw std::runtime_error(LOCATION +
"ROI index out of range.");
throw raw_file_read_error(
self, LOCATION + "ROI index out of range.");
}
// adjust for actual frames left in the file
size_t n_frames =
std::min(num_frames, self.total_frames() - self.tell());
std::min(num_frames, self.tell() < self.total_frames()
? self.total_frames() - self.tell()
: size_t{0});
if (n_frames == 0) {
throw std::runtime_error("No frames left in file");
throw raw_file_read_error(self, "No frames left in file");
}
std::vector<size_t> shape{
n_frames, self.roi_geometries(roi_index).pixels_y(),
@@ -306,4 +324,4 @@ void define_raw_file_io_bindings(py::module &m) {
.def_property_readonly("n_modules", &RawFile::n_modules)
.def_property_readonly("n_modules_in_roi", &RawFile::n_modules_in_roi)
.def_property_readonly("num_rois", &RawFile::num_rois);
}
}
+238 -2
View File
@@ -1,8 +1,244 @@
# SPDX-License-Identifier: MPL-2.0
import pytest
from aare import RawFile, ROI, UDPPortPosition
import json
from aare import File, RawFile, RawSubFile, DetectorType, ROI, UDPPortPosition
import numpy as np
@pytest.fixture
def small_raw_file(tmp_path):
master_path = tmp_path / "run_master_0.json"
master_path.write_text(json.dumps({
"Version": 7.2,
"Detector Type": "Jungfrau",
"Timing Mode": "auto",
"Geometry": {"x": 1, "y": 1},
"Image Size in bytes": 12,
"Pixels": {"x": 3, "y": 2},
"Max Frames Per File": 1,
"Total Frames": 2,
"Frames in File": 2,
"Frame Padding": 1,
"Frame Discard Policy": "nodiscard",
}))
for index in range(2):
data_path = tmp_path / f"run_d0_f{index}_0.raw"
# A detector header followed by six uint16 pixels.
data_path.write_bytes(bytes(112) + np.arange(6, dtype=np.uint16).tobytes())
return master_path
@pytest.mark.parametrize("method, args, kwargs", [
("read_frame", (), {}),
("read_n", (1,), {}),
("read_roi", (0,), {}),
("read_rois", (), {}),
("read_n_with_roi", (1,), {"roi_index": 0}),
("frame_number", (2,), {}),
])
def test_raw_read_error_context(small_raw_file, method, args, kwargs):
reader = RawFile(small_raw_file)
reader.seek(2)
with pytest.raises(RuntimeError) as error:
getattr(reader, method)(*args, **kwargs)
assert "frame index 2" in str(error.value)
assert str(small_raw_file) in str(error.value)
assert ".raw" not in str(error.value)
if method == "read_frame":
assert "file contains 2 frames (indices are zero-based)" in str(error.value)
@pytest.mark.parametrize("frame_index", [2, 17])
def test_generic_raw_read_error_context(small_raw_file, frame_index):
reader = File(small_raw_file)
with pytest.raises(RuntimeError) as error:
reader.read_frame(frame_index)
assert f"frame index {frame_index}" in str(error.value)
assert str(small_raw_file) in str(error.value)
@pytest.mark.parametrize("raw_subfile", [False, True])
def test_truncated_raw_read_error_context(small_raw_file, raw_subfile):
data_path = small_raw_file.parent / "run_d0_f1_0.raw"
if raw_subfile:
reader = RawSubFile(
small_raw_file.parent / "run_d0_f0_0.raw",
DetectorType.Jungfrau, 2, 3, 16,
)
else:
reader = RawFile(small_raw_file)
data_path.write_bytes(bytes(112))
reader.seek(1)
with pytest.raises(RuntimeError) as error:
reader.read_frame()
assert "frame index 1" in str(error.value)
assert str(data_path) in str(error.value)
assert "End of file" in str(error.value)
assert "master" not in str(error.value)
assert "RawSubFile.cpp" not in str(error.value)
@pytest.mark.parametrize("method, args", [
("read_frame", ()),
("read_n", (1,)),
("read_roi", (0,)),
("frame_number", (1,)),
])
def test_short_raw_subfile_sets_read_bounds(small_raw_file, method, args):
(small_raw_file.parent / "run_d0_f1_0.raw").unlink()
reader = RawFile(small_raw_file)
reader.seek(1)
with pytest.raises(RuntimeError) as error:
getattr(reader, method)(*args)
assert reader.total_frames == 1
assert "frame index 1" in str(error.value)
assert str(small_raw_file) in str(error.value)
assert ".raw" not in str(error.value)
@pytest.mark.parametrize("counts, master_count, warn", [
([2], 2, False),
([2, 2], 2, False),
([2, 2], 0, True),
([2, 2], 1, True),
([2, 2], 5, True),
([1, 2], 2, True),
([2, 1], 2, True),
([1, 2], 1, True),
([0, 2], 2, True),
([0, 0], 0, False),
])
def test_raw_subfile_frame_counts(small_raw_file, capfd, counts, master_count, warn):
metadata = json.loads(small_raw_file.read_text())
metadata["Geometry"]["y"] = len(counts)
metadata["Frames in File"] = master_count
metadata["Total Frames"] = 1000
small_raw_file.write_text(json.dumps(metadata))
data = (small_raw_file.parent / "run_d0_f0_0.raw").read_bytes()
for module, count in enumerate(counts):
for index in range(2):
path = small_raw_file.parent / f"run_d{module}_f{index}_0.raw"
if index < count:
path.write_bytes(data)
elif index == 0:
path.write_bytes(b"")
else:
path.unlink(missing_ok=True)
reader = RawFile(small_raw_file)
warning = capfd.readouterr().err
if warn:
assert warning.count("WARNING") == 1
assert str(small_raw_file) in warning
else:
assert "WARNING" not in warning
assert reader.total_frames == min(counts)
assert len(reader) == min(counts)
assert reader.master.frames_in_file == master_count
if min(counts):
_, frames = reader.read()
assert frames.shape == (min(counts), 2 * len(counts), 3)
reader.seek(0)
_, frames = reader.read_n(10)
assert len(frames) == min(counts)
assert reader.frame_number(min(counts) - 1) == 0
with pytest.raises(RuntimeError):
reader.frame_number(min(counts))
reader.seek(min(counts))
with pytest.raises(RuntimeError):
reader.read_frame()
assert "WARNING" not in capfd.readouterr().err
@pytest.mark.parametrize("short_module", [0, 1])
def test_raw_frame_count_across_rois(small_raw_file, short_module):
metadata = json.loads(small_raw_file.read_text())
metadata.update({
"Version": 8.0,
"Geometry": {"x": 1, "y": 2},
"Image Size": 12,
"Receiver Rois": [
{"xmin": 0, "xmax": 2, "ymin": 0, "ymax": 1},
{"xmin": 0, "xmax": 2, "ymin": 2, "ymax": 3},
],
})
small_raw_file.write_text(json.dumps(metadata))
for index in range(2):
data = (small_raw_file.parent / f"run_d0_f{index}_0.raw").read_bytes()
(small_raw_file.parent / f"run_d1_f{index}_0.raw").write_bytes(data)
(small_raw_file.parent / f"run_d{short_module}_f1_0.raw").unlink()
reader = RawFile(small_raw_file)
assert reader.total_frames == 1
_, rois = reader.read_rois()
assert len(rois) == 2
assert all(roi.shape == (2, 3) for roi in rois)
for roi_index in range(2):
reader.seek(0)
_, frames = reader.read_n_with_roi(10, roi_index=roi_index)
assert frames.shape == (1, 2, 3)
with pytest.raises(RuntimeError):
reader.read_roi(roi_index)
@pytest.mark.parametrize("disabled_port", [0, 1])
def test_raw_frame_count_with_disabled_udp_port(small_raw_file, disabled_port):
metadata = json.loads(small_raw_file.read_text())
metadata.update({
"Geometry": {"x": 1, "y": 2},
"Frames in File": 0,
"Number of UDP Interfaces": 2,
"UDP Ports Type": ["bottom", "top"],
"UDP Ports Disabled": [disabled_port],
})
small_raw_file.write_text(json.dumps(metadata))
data = (small_raw_file.parent / "run_d0_f0_0.raw").read_bytes()
for index in range(2):
(small_raw_file.parent / f"run_d0_f{index}_0.raw").unlink()
path = small_raw_file.parent / f"run_d{1 - disabled_port}_f{index}_0.raw"
path.write_bytes(data)
reader = RawFile(small_raw_file)
assert reader.total_frames == 2
assert reader.master.frames_in_file == 0
assert reader.frame_number(1) == 0
_, frames = reader.read()
assert frames.shape == (2, 2, 3)
def test_raw_frame_count_without_selected_subfiles(small_raw_file):
metadata = json.loads(small_raw_file.read_text())
metadata.update({"Version": 8.0, "Image Size": 12, "Receiver Rois": []})
small_raw_file.write_text(json.dumps(metadata))
with pytest.raises(RuntimeError, match="No raw subfiles selected"):
RawFile(small_raw_file)
@pytest.mark.parametrize("lagging_module", [0, 1])
def test_raw_synchronization_error_identifies_subfile(
small_raw_file, lagging_module
):
metadata = json.loads(small_raw_file.read_text())
metadata["Geometry"]["y"] = 2
small_raw_file.write_text(json.dumps(metadata))
for index in range(2):
data = (small_raw_file.parent / f"run_d0_f{index}_0.raw").read_bytes()
(small_raw_file.parent / f"run_d1_f{index}_0.raw").write_bytes(data)
ahead_path = small_raw_file.parent / f"run_d{1 - lagging_module}_f1_0.raw"
with ahead_path.open("r+b") as output:
output.write(np.array([1], dtype=np.uint64).tobytes())
reader = RawFile(small_raw_file)
reader.seek(1)
with pytest.raises(RuntimeError) as error:
reader.read_frame()
assert "frame index 1" in str(error.value)
assert "synchroniz" in str(error.value)
lagging_path = small_raw_file.parent / f"run_d{lagging_module}_f1_0.raw"
assert str(lagging_path) in str(error.value)
@pytest.mark.withdata
def test_read_rawfile_with_roi_spanning_over_one_module(test_data_path):
@@ -147,4 +383,4 @@ def test_read_eiger_udp_port_disabled(test_data_path):
rois = f.master.rois
assert len(rois) == 2
assert rois[0] == ROI(0, 512, 0, 512)
assert rois[1] == ROI(1024, 1536, 0, 512)
assert rois[1] == ROI(1024, 1536, 0, 512)