Merge agipd-mask-speed into rc178: XFEL per-frame mask read outside the HDF5 lock

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-22 18:16:42 +02:00
commit a6835fe9d3
3 files changed
+71 -23

No files matched your search

+58 -20
View File
@@ -425,6 +425,28 @@ JFJochXFELReader::Layout JFJochXFELReader::ReadLayout(HDF5ReadOnlyFile &file, co
return l;
}
// A per-frame mask is read as the frames are where it is stored in plain unsigned integers - read as
// stored, they are what HDF5 would convert them to - of the frame's size, in every file the way the
// first file stores it. Any other is read by HDF5.
JFJochXFELReader::Layout JFJochXFELReader::FrameMaskLayout(const xfel::Mask &m) const {
Layout layout;
for (size_t i = 0; i < frames_.size(); i++) {
if (i > 0 && frames_[i].file == frames_[i - 1].file)
continue; // a file's first frame stands for all of its frames, as for the frames themselves
const Frame mask{frames_[i].file, ReplaceEvent(m.data, frames_[i].group), frames_[i].index, frames_[i].group};
if (!FileAt(mask.file).Exists(mask.dataset))
return {};
const Layout l = ReadLayout(FileAt(mask.file), mask);
if (i == 0)
layout = l;
else if (!(l == layout))
return {};
}
if (layout.stored_float || layout.stored_signed || layout.height != frame_height_ || layout.width != frame_width_)
return {};
return layout;
}
void JFJochXFELReader::Open(const std::vector<std::string> &files) {
files_ = files;
geom_.masks.insert(geom_.masks.end(), extra_masks_.begin(), extra_masks_.end());
@@ -515,11 +537,13 @@ void JFJochXFELReader::Place() {
{
std::unique_lock ul(hdf5_mutex);
frame_masks_.clear();
frame_mask_layouts_.clear();
for (const auto &m: geom_.masks) {
std::vector<uint32_t> v;
if (m.file.empty() && (m.data.find('%') != std::string::npos
|| FileAt(0).GetDimension(m.data).size() == 3)) {
frame_masks_.push_back(m); // one per shot or one plane per frame: applied as each is read
frame_mask_layouts_.push_back(FrameMaskLayout(m));
continue;
}
if (!m.file.empty()) {
@@ -775,7 +799,8 @@ void JFJochXFELReader::Close() {
}
template <class Buffer>
bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_compressed) const {
bool JFJochXFELReader::ReadStored(const Frame &frame, const Layout &layout, Buffer &stored,
bool keep_compressed) const {
const std::vector<hsize_t> start = frame.index >= 0
? std::vector<hsize_t>{static_cast<hsize_t>(frame.index), 0, 0}
: std::vector<hsize_t>{0, 0};
@@ -788,7 +813,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
std::unique_lock ul(hdf5_mutex);
HDF5DataSet dataset(FileAt(frame.file), frame.dataset);
unsigned filter_mask = 0;
if (layout_.direct_chunks) {
if (layout.direct_chunks) {
haddr_t addr = HADDR_UNDEF;
hsize_t size = 0;
if (H5Dget_chunk_info_by_coord(dataset.GetID(), start.data(), &filter_mask, &addr, &size) < 0
@@ -800,8 +825,8 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
}
// A chunk stored with one of its filters skipped (its filter mask) is not the encoding decoded
// below: HDF5 reads it, as it reads a frame that is not stored as one chunk.
if (!layout_.direct_chunks || filter_mask != 0) {
if (layout_.plain_type)
if (!layout.direct_chunks || filter_mask != 0) {
if (layout.plain_type)
dataset.ReadVectorToU8(stored, start, count);
else {
std::vector<float> v(frame_width_ * frame_height_);
@@ -814,7 +839,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
raw_file = RawFileAt(frame.file);
address += user_block_[frame.file]; // HDF5's chunk addresses count from the end of the user block
}
if (keep_compressed && !layout_.deflated) {
if (keep_compressed && !layout.deflated) {
stored.resize(chunk_size);
raw_file->ReadAt(stored.data(), stored.size(), address);
return true;
@@ -824,19 +849,19 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
raw_file->ReadAt(compressed.data(), compressed.size(), address);
const size_t n = frame_width_ * frame_height_;
const size_t size = layout_.stored_size;
if (layout_.deflated) {
if (!layout_.shuffled)
const size_t size = layout.stored_size;
if (layout.deflated) {
if (!layout.shuffled)
stored.resize(n * size);
else
shuffled.resize(n * size);
uint8_t *inflated = layout_.shuffled ? shuffled.data() : stored.data();
uint8_t *inflated = layout.shuffled ? shuffled.data() : stored.data();
uLongf inflated_size = n * size;
if (uncompress(inflated, &inflated_size, compressed.data(), compressed.size()) != Z_OK
|| inflated_size != n * size)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Cannot inflate " + frame.dataset + " of " + files_[frame.file]);
if (layout_.shuffled) {
if (layout.shuffled) {
stored.resize(n * size);
for (size_t i = 0; i < n; i++)
for (size_t b = 0; b < size; b++)
@@ -844,7 +869,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
}
} else {
stored.resize(n * size);
JFJochDecompressPtr(stored.data(), layout_.algorithm, compressed.data(), compressed.size(), n, size);
JFJochDecompressPtr(stored.data(), layout.algorithm, compressed.data(), compressed.size(), n, size);
}
return false;
}
@@ -884,24 +909,37 @@ CompressedImage JFJochXFELReader::DecodeInto(int64_t image_number, Buffer &buffe
const bool as_stored = blocks_.empty() && layout_.plain_type && !layout_.stored_float && layout_.stored_signed
&& adu_per_photon_ == 1.0f;
if (as_stored && frame_masks_.empty()) {
const bool compressed = ReadStored(frame, buffer, true);
const bool compressed = ReadStored(frame, layout_, buffer, true);
return CompressedImage(buffer.data(), buffer.size(), width_, height_, stored_mode,
compressed ? layout_.algorithm : CompressionAlgorithm::NO_COMPRESSION);
}
thread_local std::vector<uint8_t> stored;
ReadStored(frame, stored, false);
ReadStored(frame, layout_, stored, false);
// A pixel a per-frame mask marks in this frame is handed out as no measurement.
std::vector<uint8_t> masked;
if (!frame_masks_.empty()) {
masked.assign(frame_width_ * frame_height_, 0);
std::unique_lock ul(hdf5_mutex);
for (const auto &m: frame_masks_) {
const auto path = ReplaceEvent(m.data, frame.group);
const auto v = frame.index >= 0
? FileAt(frame.file).ReadVector<uint32_t>(path, {static_cast<hsize_t>(frame.index), 0, 0},
{1, frame_height_, frame_width_})
: FileAt(frame.file).ReadVector<uint32_t>(path, {0, 0}, {frame_height_, frame_width_});
for (size_t i = 0; i < frame_masks_.size(); i++) {
const auto &m = frame_masks_[i];
const auto &mask_layout = frame_mask_layouts_[i];
const Frame mask{frame.file, ReplaceEvent(m.data, frame.group), frame.index, frame.group};
std::vector<uint32_t> v(frame_width_ * frame_height_);
if (mask_layout.plain_type) {
thread_local std::vector<uint8_t> mask_stored;
ReadStored(mask, mask_layout, mask_stored, false);
if (mask_layout.stored_size == 2) {
const auto *p = reinterpret_cast<const uint16_t *>(mask_stored.data());
std::copy(p, p + v.size(), v.begin());
} else
std::memcpy(v.data(), mask_stored.data(), v.size() * sizeof(uint32_t));
} else {
std::unique_lock ul(hdf5_mutex);
v = frame.index >= 0
? FileAt(frame.file).ReadVector<uint32_t>(mask.dataset, {static_cast<hsize_t>(frame.index), 0, 0},
{1, frame_height_, frame_width_})
: FileAt(frame.file).ReadVector<uint32_t>(mask.dataset, {0, 0}, {frame_height_, frame_width_});
}
MarkBad(m, v, masked);
}
}
+8 -3
View File
@@ -96,6 +96,9 @@ class JFJochXFELReader : public JFJochReader {
bool operator==(const Layout &) const = default;
};
Layout layout_;
// For each of frame_masks_, how it is stored where it is read as the frames are (ReadStored): plain
// unsigned integers of the frame's size, stored alike in every file; plain_type false where HDF5 reads it.
std::vector<Layout> frame_mask_layouts_;
// Positional-read handles on the files, opened as first needed (under hdf5_mutex), with the size of
// each file's user block, which HDF5 chunk addresses count from. Like open_files_, only the most
// recently used MAX_OPEN_FILES are held (raw_order_); a reader still using a released one keeps it.
@@ -103,6 +106,7 @@ class JFJochXFELReader : public JFJochReader {
mutable std::vector<uint64_t> user_block_;
mutable std::vector<size_t> raw_order_;
static Layout ReadLayout(HDF5ReadOnlyFile &file, const Frame &frame);
Layout FrameMaskLayout(const xfel::Mask &m) const; // under hdf5_mutex
// The files, in the order given, every frame in them, and the run's mean of every value the geometry
// takes from the files.
@@ -116,10 +120,11 @@ class JFJochXFELReader : public JFJochReader {
int64_t image_number,
bool update_dataset) override;
// The frame's pixels as stored (plain_type), or as float, into stored; with keep_compressed, a chunk
// decoded by CompressedImage (not deflated) is left as stored in the file, and true returned.
// The frame's pixels, stored as layout says, as stored (plain_type), or as float, into stored; with
// keep_compressed, a chunk decoded by CompressedImage (not deflated) is left as stored in the file, and
// true returned.
template <class Buffer>
bool ReadStored(const Frame &frame, Buffer &stored, bool keep_compressed) const;
bool ReadStored(const Frame &frame, const Layout &layout, Buffer &stored, bool keep_compressed) const;
// The stored pixels as the image handed out (as stored, or placed), as Out, each through convert; a pixel masked marks is err.
template <class Out, class In, class Buffer, class Convert>
void PlaceInto(const In *in, const std::vector<uint8_t> &masked, Out err, Buffer &buffer, Convert convert) const;
+5
View File
@@ -280,6 +280,11 @@ TEST_CASE("JFJochXFELReader_GeometryMasks", "[portable]") {
std::vector<uint16_t> mask(2 * 24, 0);
mask[1 * 6 + 1] = 0x20; // frame 0 only, (fs 1, ss 1): a bit Cheetah's own set does not include
WriteFrames(f, "entry_1/data_1/mask", H5T_NATIVE_UINT16, mask.data(), {2, 4, 6});
// Frame 1's mask chunk is stored with both its filters skipped: HDF5 reads that one.
const hid_t ds = H5Dopen2(f, "entry_1/data_1/mask", H5P_DEFAULT);
const hsize_t offset[3] = {1, 0, 0};
REQUIRE(H5Dwrite_chunk(ds, H5P_DEFAULT, 0x3, offset, 24 * sizeof(uint16_t), mask.data() + 24) >= 0);
H5Dclose(ds);
const std::string det = "entry_1/instrument_1/detector_1/";
std::vector<uint16_t> shared(24, 0);
shared[3 * 6 + 5] = 8; // (fs 5, ss 3) dead to Cheetah